diff --git a/.helmignore b/.helmignore index fbdd321f..003f12b7 100644 --- a/.helmignore +++ b/.helmignore @@ -24,3 +24,6 @@ *.tmproj .circleci/ + +doc/ +.github/ \ No newline at end of file diff --git a/Chart.yaml b/Chart.yaml index 00e23c75..33b1d992 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,8 +1,8 @@ --- apiVersion: v2 name: nifi -version: 0.7.9 -appVersion: 1.12.1 +version: 1.0.0 +appVersion: 1.14.0 description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems. keywords: - nifi @@ -38,3 +38,7 @@ dependencies: - name: ca version: 1.0.1 condition: ca.enabled + - name: openldap + version: ~1.2.4 + repository: https://charts.helm.sh/stable + condition: openldap.enabled diff --git a/README.md b/README.md index f278cdb1..9f9def16 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Introduction -This [Helm](https://helm.sh/) chart installs [Apache NiFi](https://nifi.apache.org/) in a [Kubernetes](https://kubernetes.io/) cluster. +This [Helm](https://helm.sh/) chart installs [Apache NiFi](https://nifi.apache.org/) 1.14.0 in a [Kubernetes](https://kubernetes.io/) cluster. ## Prerequisites @@ -40,10 +40,7 @@ The following items can be set via `--set` flag during installation or configure #### Configure authentication -- You first need a secure cluster which can be accomplished by enabling the built-in CA nifi-toolkit container (`ca.enabled` to true). By default, a secure nifi cluster uses certificate based authentication but you can optionally enable `ldap` or `oidc`. See the configuration section for more details. - -:warning: This feature is quite new. Please open an issue if you encounter a problem. -It seems that versions from 0.6.1 include some bugs for authentications. Please use version 0.6.0 of the chart until it is fixed. +- By default, the authentication is a `Single-User` authentication. You can optionally enable `ldap` or `oidc` to provide an external authentication. See the [configuration section](README.md#configuration) or [doc](doc/) folder for more details. #### Use custom processors @@ -78,15 +75,7 @@ helm install my-release cetic/nifi ### Install from local clone -```bash -git clone https://github.com/cetic/helm-nifi.git nifi -cd nifi -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo add dysnix https://dysnix.github.io/charts/ -helm repo update -helm dep up -helm install nifi . -``` +You will find how to perform an installation from a local clone on this [page](doc/INSTALLATION.md). ## Uninstallation @@ -106,7 +95,7 @@ The following table lists the configurable parameters of the nifi chart and the | `replicaCount` | Number of nifi nodes | `1` | | **Image** | | `image.repository` | nifi Image name | `apache/nifi` | -| `image.tag` | nifi Image tag | `1.12.1` | +| `image.tag` | nifi Image tag | `1.14.0` | | `image.pullPolicy` | nifi Image pull policy | `IfNotPresent` | | `image.pullSecret` | nifi Image pull secret | `nil` | | **SecurityContext** | @@ -131,20 +120,23 @@ The following table lists the configurable parameters of the nifi chart and the | `properties.httpPort` | web properties HTTP port | `8080` | | `properties.httpsPort` | web properties HTTPS port | `null` | | `properties.clusterPort` | cluster node port | `6007` | -| `properties.clusterSecure` | cluster nodes secure mode | `false` | -| `properties.needClientAuth` | nifi security client auth | `false` | | `properties.provenanceStorage` | nifi provenance repository max storage size | `8 GB` | | `properties.siteToSite.secure` | Site to Site properties Secure mode | `false` | | `properties.siteToSite.port` | Site to Site properties Secure port | `10000` | -| `properties.siteToSite.authorizer` | | `managed-authorizer` | | `properties.safetyValve` | Map of explicit 'property: value' pairs that overwrite other configuration | `nil` | | `properties.customLibPath` | Path of the custom libraries folder | `nil` | -| **nifi user authentication** | +| `properties.webProxyHost` | Proxy to access to Nifi through the cluster ip address | `Port:30236` +| **[Authentication](/doc/USERMANAGEMENT.md)** | +| **Single-user authentication** | Automatically disabled if OIDC or LDAP enabled +| `auth.singleUser.username` | Single user identity | `username` | +| `auth.singleUser.password` | Single user password | `changemechangeme` | +| **Ldap authentication** | | `auth.admin` | Default admin identity | ` CN=admin, OU=NIFI` | | `auth.ldap.enabled` | Enable User auth via ldap | `false` | | `auth.ldap.host` | ldap hostname | `ldap://:` | | `auth.ldap.searchBase` | ldap searchBase | `CN=Users,DC=example,DC=com` | | `auth.ldap.searchFilter` | ldap searchFilter | `CN=john` | +| **Oidc authentication** | `auth.oidc.enabled` | Enable User auth via oidc | `false` | | `auth.oidc.discoveryUrl` | oidc discover url | `https:///.well-known/openid-configuration` | | `auth.oidc.clientId` | oidc clientId | `nil` | diff --git a/configs/authorizers-empty.xml b/configs/authorizers-empty.xml index 0d3b04d8..a7e707dc 100644 --- a/configs/authorizers-empty.xml +++ b/configs/authorizers-empty.xml @@ -1,3 +1,8 @@ +{{- $replicas := int .Values.replicaCount }} +{{- $chart := .Chart.Name }} +{{- $release := .Release.Name }} +{{- $fullname := include "apache-nifi.fullname" . }} +{{- $namespace := .Release.Namespace }} - + {{- end}} - + {{- end}} - - + {{- end}} + \ No newline at end of file diff --git a/configs/authorizers.xml b/configs/authorizers.xml index 72da629b..e3b843d9 100644 --- a/configs/authorizers.xml +++ b/configs/authorizers.xml @@ -45,7 +45,7 @@ file-user-group-provider org.apache.nifi.authorization.FileUserGroupProvider - ./auth-conf/users.xml + ./conf/users.xml {{- range $i := until $replicas }} CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI @@ -221,7 +221,7 @@ file-access-policy-provider org.apache.nifi.authorization.FileAccessPolicyProvider file-user-group-provider - ./auth-conf/authorizations.xml + ./conf/authorizations.xml {{- if .Values.auth.ldap.enabled}} {{.Values.auth.ldap.admin}} {{- else }} @@ -279,4 +279,4 @@ {{- end}} - + \ No newline at end of file diff --git a/configs/nifi.properties b/configs/nifi.properties index c2f74521..5f5f63d2 100644 --- a/configs/nifi.properties +++ b/configs/nifi.properties @@ -122,7 +122,7 @@ nifi.components.status.snapshot.frequency=1 min # Site to Site properties nifi.remote.input.host= -nifi.remote.input.secure={{.Values.properties.clusterSecure}} +nifi.remote.input.secure=true nifi.remote.input.socket.port={{.Values.properties.siteToSite.port}} nifi.remote.input.http.enabled=true nifi.remote.input.http.transaction.ttl=30 sec @@ -131,13 +131,7 @@ nifi.remote.contents.cache.expiration=30 secs # web properties # nifi.web.war.directory=./lib nifi.web.proxy.host={{.Values.properties.webProxyHost}} - {{if .Values.properties.clusterSecure}} -nifi.web.http.port= nifi.web.https.port={{.Values.properties.httpsPort}} - {{else}} -nifi.web.http.port={{.Values.properties.httpPort}} -nifi.web.https.port= - {{end}} nifi.web.http.host= nifi.web.http.network.interface.default= nifi.web.https.host={{.Values.properties.webHttpsHost}} @@ -147,9 +141,9 @@ nifi.web.jetty.threads=200 # nifi.web.proxy.context.path= # security properties # -nifi.sensitive.props.key= +nifi.sensitive.props.key={{.Values.properties.sensitiveKey}} nifi.sensitive.props.key.protected= -nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.algorithm={{ .Values.properties.alogorithm }} nifi.sensitive.props.provider=BC nifi.sensitive.props.additional.keys= @@ -164,26 +158,22 @@ nifi.security.truststorePasswd={{.Values.auth.SSL.truststorePasswd}} proxiedEntity={{.Values.auth.ldap.admin}} nifi.security.user.authorizer=file-provider nifi.security.needClientAuth={{.Values.properties.needClientAuth}} -{{else}} -nifi.security.keystore= -nifi.security.keystoreType= +nifi.security.user.login.identity.provider=ldap-provider +{{end}} + +{{if .Values.auth.oidc.enabled}} +nifi.security.keystore=/opt/nifi/nifi-current/conf/keystore.p12 +nifi.security.keystoreType=PKCS12 nifi.security.keystorePasswd= nifi.security.keyPasswd= -nifi.security.truststore= -nifi.security.truststoreType= +nifi.security.truststore=/opt/nifi/nifi-current/conf/truststore.p12 +nifi.security.truststoreType=PKCS12 nifi.security.truststorePasswd= nifi.security.needClientAuth={{.Values.properties.needClientAuth}} -nifi.security.user.authorizer={{.Values.properties.authorizer}} +nifi.security.user.authorizer=managed-authorizer {{end}} -{{if .Values.auth.ldap.enabled}} -nifi.security.user.login.identity.provider=ldap-provider -{{else}} -nifi.security.user.login.identity.provider= -{{end}} -nifi.security.ocsp.responder.url= -nifi.security.ocsp.responder.certificate= - {{if .Values.auth.oidc.enabled}} +{{if .Values.auth.oidc.enabled}} # OpenId Connect SSO Properties # nifi.security.user.oidc.discovery.url={{.Values.auth.oidc.discoveryUrl}} nifi.security.user.oidc.connect.timeout=5 secs @@ -193,7 +183,7 @@ nifi.security.user.oidc.client.secret={{.Values.auth.oidc.clientSecret}} nifi.security.user.oidc.preferred.jwsalgorithm= nifi.security.user.oidc.claim.identifying.user={{.Values.auth.oidc.claimIdentifyingUser}} nifi.security.user.oidc.additional.scopes={{.Values.auth.oidc.additionalScopes}} - {{end}} +{{end}} # Apache Knox SSO Properties # diff --git a/configs/state-management.xml b/configs/state-management.xml index e6dbce4c..111b516f 100644 --- a/configs/state-management.xml +++ b/configs/state-management.xml @@ -3,7 +3,7 @@ local-provider org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider - ../data/state/local + ./state/local false 16 2 mins diff --git a/doc/INSTALLATION.md b/doc/INSTALLATION.md new file mode 100644 index 00000000..7ad1044b --- /dev/null +++ b/doc/INSTALLATION.md @@ -0,0 +1,47 @@ +Installation +============= + + +### Install from local clone + +1. **Clone the repo** + +```bash +git clone https://github.com/cetic/helm-nifi.git +cd helm-nifi +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add dysnix https://dysnix.github.io/charts/ +helm repo update +helm dep up +``` +2. **Set a sensitiveKey** + +In 1.14.0 version, Nifi needs a sensitiveKey to encrypt sensitive information. This key can be setted in the `values.yaml` file: + +```` +properties: + sensitiveKey: changeMechangeMe +```` + +3. **Configure a user authentication** + +This helm chart provides three types of authentication: Single User, LDAP and OIDC. + +You can find how to configure these authentications on this [page](https://github.com/cetic/helm-nifi/tree/feature/nifi_1.14.0/doc/USERMANAGER.md). + +4. **Install Nifi** + +To install Nifi, run this command: + +```bash +helm install nifi . +``` +5. **Access Nifi** + +If you let the Nifi service in ClusterIP mode, you cannot reach Nifi from the outside of the cluster. To fix that, you have to make a port forwarding to access Nifi from the localhost. To do that, run the command below: + +```` +kubectl port-forward service/nifi 8443:8443 +```` + +Now you can access to Nifi with a browser by typing the address: `https://localhost:8443` \ No newline at end of file diff --git a/doc/KEYCLOAK.md b/doc/KEYCLOAK.md new file mode 100644 index 00000000..0ab59431 --- /dev/null +++ b/doc/KEYCLOAK.md @@ -0,0 +1,109 @@ +KeyCloak +============= + +Example OIDC configuration with Keycloak provider + +## 0. Pull, extract and install the Keycloak helm chart +``` +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm pull bitnami/keycloak +tar -xzvf keycloak-*.tgz +cd keycloak/ +``` +Keycloak is first deployed with one admin user that we need to configure before deployment, to do so we need to pass the username/password in the `values.yaml` file like follows: + +``` +auth: + adminUser: admin + adminPassword: password +``` +Launching installation + + +``` +helm dep up +helm install my-release bitnami/keycloak +``` +## 1. Configure keycloak + + Keycloak logo + +> "Keycloak is an open source Identity and Access Management solution aimed at modern applications and services. It makes it easy to secure applications and services with little to no code." + +Once deployed we can access the user interface and log in: + +Keycloak login + +We click on **administration console** then we log in using the credentials we configured above. + +Keycloak home + +## Create realm + +When you log in to the admin console, you work in a realm, which is a space where you manage objects. Two types of realms exist: + +* **Master realm** - This realm was created for you when you first started Keycloak. It contains the admin account you created at the first login. You use this realm only to create other realms. + +* **Other realms** - These realms are created by the admin in the master realm. In these realms, administrators create users and applications. The applications are owned by the users. + +Keycloak realms + +To create a realm, head to the Master menu, click `Add Realm`. When you are logged in to the master realm, this menu lists all other realms, then type for example `devops` in the Name field to name our new realm devops. + +Keycloak add realm + +When we click `Create`, the main admin console page opens with realm set to devops, now we can switch between managing the master realm and the realm we just created by clicking entries in the `Select realm` drop-down list. + +Keycloak select realm + + +## Create Nifi client + +To create clients we first click **Clients** in the **left side menu** to open the Clients page. + +Keycloak create client + +On the right side, we click `Create` and then on the `Add Client` dialog, we create a client called `Nifi` by filling the fields as follows: + +* Client ID: `Nifi` +* Root URL: `\`, for this example our Nifi adress is https://localhost:8443 (don't forget the "s" of "https") + +Keycloak create client + +Once the client is created, we open the client configuration and change the **access type** to **confidential** from public, and complete the rest of the fields as shown below assuming our Nifi address is https://localhost:8443, then we **Save the config**. + + +Keycloak Nifi client created + + +Now we open the client Nifi again, go to **credentials tag** and copy the `client id` and `secret` because we are going to need them to configure Nifi later. + +Keycloak get Nifi credentials + +## Create user + + +In the devops realm, we need to create a new user and a temporary password for that new user, we head to the left menu, click Users to open the user list page. + +Create user + + +On the right side of the empty user list, click Add User to open the Add user page. + +Create user + + +we enter a name in the Username field (this is the only required field), then we flip the Email Verified switch to On and click Save. + +Create user + + + +The management page for the new user opens, we Click the **Credentials tab** to set a temporary password for the new user, we type a new password and confirm it. + +Create user + + +then we Click **Set Password** to set the user password to the new one we specified. + diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..dbb9af2f --- /dev/null +++ b/doc/README.md @@ -0,0 +1,6 @@ +Nifi installation & configuration +========== + +* [Nifi installation](INSTALLATION.md) - install Nifi locally to update the chart +* [Users management](USERMANAGEMENT.md) - user identification and authorization (Single-User, OIDC, LDAP) +* [Keycloak](Keycloak.md) - example OIDC configuration with Keycloak provider diff --git a/doc/USERMANAGEMENT.md b/doc/USERMANAGEMENT.md new file mode 100644 index 00000000..2975e98a --- /dev/null +++ b/doc/USERMANAGEMENT.md @@ -0,0 +1,85 @@ +User Authentication +============= + +This helm chart provides three types of authentication: Single User, LDAP and OIDC. These three authtications can be managed essentialy from the `values.yaml` file. + + +## 1. Single User + +The Single User authentication is the default authentication in this helm chart. To login like a single user, the values below must be setted in `values.yaml` file: + +```` +singleUser: + username: username + password: changemechangeme +```` + +## 2. OIDC + +OpenID Connect (OIDC) is an open authentication protocol that profiles and extends OAuth 2.0 to add an identity layer. It can be used by an external identity provider to make authentication. + +To enable OIDC user authentication, the values below must be setted in `values.yaml` file: + +```` +oidc: + enabled: true + discoveryUrl: http://:/auth/realms//.well-known/openid-configuration + clientId: + clientSecret: +```` + +Then, you have to set a initial user to access Nifi for the first time. This user must be present in the OIDC provider. If you are the administrator, you can use your credentials. The initial user is added in `authorizers-empty.xml` file. In our example, the initial user is `john`: + +```` + + file-user-group-provider + org.apache.nifi.authorization.FileUserGroupProvider + ./conf/users.xml + + {{- range $i := until $replicas }} + john + {{- end }} + {{- if .Values.auth.ldap.enabled}} + {{.Values.auth.ldap.admin}} + {{- end}} + +```` + +```` + + file-access-policy-provider + org.apache.nifi.authorization.FileAccessPolicyProvider + file-user-group-provider + ./conf/authorizations.xml + {{- if .Values.auth.ldap.enabled}} + {{.Values.auth.ldap.admin}} + {{- else }} + john + {{- end}} + + {{- if .Values.auth.ldap.enabled}} + {{- range $i := until $replicas }} + CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI + {{- end }} + {{- end }} + + +```` + +There are a lot of ID providers that can be used to perform an OIDC authentication. In our case, we have tested that with Keycloak. You will find an example of Keycloak config on this [page](https://github.com/cetic/helm-nifi/tree/feature/nifi_1.14.0/doc/KEYCLOAK.md). + + +## 3. LDAP + +Like OIDC, LDAP (Lightweight Directory Access Protocol) provide an external authentication. If you have your own LDAP, you can use it. If not, set `openldap.enabled` to `true` in `values.yaml` file to deploy a local instance of OpenLDAP. + +To enable authentication through LDAP, set the values below in `values.yaml` file: + +```` +ldap: + enabled: true + host: ldap://: + searchBase: CN=Users,DC=example,DC=com + admin: cn=admin,dc=example,dc=be + pass: changeMe +```` diff --git a/doc/images/installation/add-realm.png b/doc/images/installation/add-realm.png new file mode 100644 index 00000000..db27d1bf Binary files /dev/null and b/doc/images/installation/add-realm.png differ diff --git a/doc/images/installation/add-user.png b/doc/images/installation/add-user.png new file mode 100644 index 00000000..920db35c Binary files /dev/null and b/doc/images/installation/add-user.png differ diff --git a/doc/images/installation/change-pass.png b/doc/images/installation/change-pass.png new file mode 100644 index 00000000..80b18ccd Binary files /dev/null and b/doc/images/installation/change-pass.png differ diff --git a/doc/images/installation/client-nifi-created.PNG b/doc/images/installation/client-nifi-created.PNG new file mode 100644 index 00000000..f858e857 Binary files /dev/null and b/doc/images/installation/client-nifi-created.PNG differ diff --git a/doc/images/installation/client-nifi.png b/doc/images/installation/client-nifi.png new file mode 100644 index 00000000..0422e968 Binary files /dev/null and b/doc/images/installation/client-nifi.png differ diff --git a/doc/images/installation/devops-realm.png b/doc/images/installation/devops-realm.png new file mode 100644 index 00000000..6862256e Binary files /dev/null and b/doc/images/installation/devops-realm.png differ diff --git a/doc/images/installation/grafana-keycloak-auth.png b/doc/images/installation/grafana-keycloak-auth.png new file mode 100644 index 00000000..28652e05 Binary files /dev/null and b/doc/images/installation/grafana-keycloak-auth.png differ diff --git a/doc/images/installation/john-doe.png b/doc/images/installation/john-doe.png new file mode 100644 index 00000000..54f548ed Binary files /dev/null and b/doc/images/installation/john-doe.png differ diff --git a/doc/images/installation/keycloak-clients.png b/doc/images/installation/keycloak-clients.png new file mode 100644 index 00000000..9c70ab60 Binary files /dev/null and b/doc/images/installation/keycloak-clients.png differ diff --git a/doc/images/installation/keycloak-first-screen.png b/doc/images/installation/keycloak-first-screen.png new file mode 100644 index 00000000..f9452389 Binary files /dev/null and b/doc/images/installation/keycloak-first-screen.png differ diff --git a/doc/images/installation/keycloak-realms.png b/doc/images/installation/keycloak-realms.png new file mode 100644 index 00000000..f0581ad6 Binary files /dev/null and b/doc/images/installation/keycloak-realms.png differ diff --git a/doc/images/installation/keycloak-ui.png b/doc/images/installation/keycloak-ui.png new file mode 100644 index 00000000..da9f5318 Binary files /dev/null and b/doc/images/installation/keycloak-ui.png differ diff --git a/doc/images/installation/nifi-credentials.PNG b/doc/images/installation/nifi-credentials.PNG new file mode 100644 index 00000000..dbb350f0 Binary files /dev/null and b/doc/images/installation/nifi-credentials.PNG differ diff --git a/doc/images/installation/users-page.png b/doc/images/installation/users-page.png new file mode 100644 index 00000000..9435fa51 Binary files /dev/null and b/doc/images/installation/users-page.png differ diff --git a/doc/images/logos/cetic.png b/doc/images/logos/cetic.png new file mode 100644 index 00000000..c66290f5 Binary files /dev/null and b/doc/images/logos/cetic.png differ diff --git a/doc/images/logos/helm.png b/doc/images/logos/helm.png new file mode 100644 index 00000000..7c83a7b2 Binary files /dev/null and b/doc/images/logos/helm.png differ diff --git a/doc/images/logos/keycloak-logo.png b/doc/images/logos/keycloak-logo.png new file mode 100644 index 00000000..c3439f48 Binary files /dev/null and b/doc/images/logos/keycloak-logo.png differ diff --git a/doc/images/logos/nifi.png b/doc/images/logos/nifi.png new file mode 100644 index 00000000..6677cd9b Binary files /dev/null and b/doc/images/logos/nifi.png differ diff --git a/templates/ingress.yaml b/templates/ingress.yaml index 28395340..9a71e5cf 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -3,7 +3,6 @@ {{- $fullName := include "apache-nifi.fullname" . -}} {{- $ingressPath := .Values.ingress.path -}} {{- $ingressHttpsPort := .Values.service.httpsPort -}} -{{- $ingressHttpPort := .Values.service.httpPort -}} apiVersion: extensions/v1beta1 kind: Ingress metadata: @@ -29,7 +28,6 @@ spec: secretName: {{ .secretName }} {{- end }} {{- end }} -{{- if .Values.properties.clusterSecure}} rules: {{- range .Values.ingress.hosts }} - host: {{ . }} @@ -40,16 +38,4 @@ spec: serviceName: {{ $fullName }} servicePort: {{ $ingressHttpsPort }} {{- end }} -{{- else}} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ . }} - http: - paths: - - path: {{ $ingressPath }} - backend: - serviceName: {{ $fullName }} - servicePort: {{ $ingressHttpPort }} - {{- end }} -{{- end }} {{- end }} diff --git a/templates/route.yaml b/templates/route.yaml index 2637cd7e..d705a44d 100644 --- a/templates/route.yaml +++ b/templates/route.yaml @@ -22,7 +22,6 @@ spec: name: {{ $fullName }} weight: 100 port: -{{- if .Values.properties.clusterSecure }} targetPort: https tls: {{- if .Values.properties.externalSecure }} @@ -31,7 +30,4 @@ spec: termination: passthrough {{- end }} insecureEdgeTerminationPolicy: Redirect -{{- else }} - targetPort: http -{{- end }} {{- end }} \ No newline at end of file diff --git a/templates/service.yaml b/templates/service.yaml index 2296af58..05a634be 100644 --- a/templates/service.yaml +++ b/templates/service.yaml @@ -16,13 +16,8 @@ spec: type: {{ .Values.headless.type }} clusterIP: None ports: -{{- if .Values.properties.clusterSecure }} - port: {{ .Values.properties.httpsPort }} name: https -{{- else}} - - port: {{ .Values.properties.httpPort }} - name: http -{{- end }} - port: {{ .Values.properties.clusterPort }} name: cluster selector: @@ -63,16 +58,10 @@ spec: {{- end }} {{- end }} ports: -{{- if .Values.properties.clusterSecure }} - port: {{ .Values.service.httpsPort }} name: https targetPort: {{ .Values.properties.httpsPort }} nodePort: {{ .Values.service.nodePort }} -{{- else }} - - port: {{ .Values.service.httpPort }} - name: http - targetPort: {{ .Values.properties.httpPort }} -{{- end }} {{- if .Values.service.processors.enabled }} {{- with .Values.service.processors.ports }} {{- range . }} diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index 075acfeb..ca8d008e 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -102,62 +102,7 @@ spec: sleep 2 done {{- end }} -{{- if .Values.ca.enabled }} - - name: cert-request - imagePullPolicy: {{ .Values.ca.image.pullPolicy | quote }} - image: "{{ .Values.ca.image.repository }}:{{ .Values.ca.image.tag }}" - command: - - bash - - -c - - | - CA_ADDRESS="{{ template "ca.server" . }}:{{ .Values.ca.service.port }}" - until echo "" | timeout -t 2 openssl s_client -connect "${CA_ADDRESS}"; do - # Checking if ca server using nifi-toolkit is up - echo "Waiting for CA to be available at ${CA_ADDRESS}" - sleep 2 - done; - cd /data/config-data - rm -rf certs - mkdir certs - cd certs - - # Generate certificate for server with webProxyHost or service name as alternate names to access nifi web ui - ${NIFI_TOOLKIT_HOME}/bin/tls-toolkit.sh client \ - -c "{{ template "ca.server" . }}" \ - -t {{ .Values.ca.token }} \ -{{- if .Values.properties.webProxyHost }} - --subjectAlternativeNames {{ .Values.properties.webProxyHost }}, $(hostname -f) \ -{{- else }} - --subjectAlternativeNames {{ template "apache-nifi.fullname" . }}.{{ .Release.Namespace }}.svc \ -{{- end }} - -D "CN=$(hostname -f), OU=NIFI" \ - -p {{ .Values.ca.service.port }} - - # Generate client certificate for browser with webProxyHost or service name as alternate names to access nifi web ui - mkdir -p /data/config-data/certs/admin - cd /data/config-data/certs/admin - ${NIFI_TOOLKIT_HOME}/bin/tls-toolkit.sh client \ - -c "{{ template "ca.server" . }}" \ - -t {{ .Values.ca.token }} \ -{{- if .Values.properties.webProxyHost }} - --subjectAlternativeNames {{ .Values.properties.webProxyHost }} \ -{{- else }} - --subjectAlternativeNames {{ template "apache-nifi.fullname" . }}.{{ .Release.Namespace }}.svc \ -{{- end }} - -p {{ .Values.ca.service.port }} \ - -D "CN={{ .Values.ca.admin.cn }}, OU=NIFI" \ - -T PKCS12 - - export PASS=$(jq -r .keyStorePassword config.json) - - openssl pkcs12 -in "keystore.pkcs12" -out "key.pem" -nocerts -nodes -password "env:PASS" - openssl pkcs12 -in "keystore.pkcs12" -out "crt.pem" -clcerts -nokeys -password "env:PASS" - openssl pkcs12 -in "keystore.pkcs12" -out "keystore.jks" -clcerts -nokeys -password "env:PASS" - volumeMounts: - - name: "config-data" - mountPath: /data/config-data -{{- end }} {{- range $key, $value := .Values.initContainers }} - name: {{ $key }} {{ toYaml $value | indent 8 }} @@ -173,6 +118,7 @@ spec: - name: server imagePullPolicy: {{ .Values.image.pullPolicy | quote }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +{{- if or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled) }} command: - bash - -ce @@ -186,7 +132,6 @@ spec: echo ${1}=${2} >> ${target_file} fi } - mkdir -p ${NIFI_HOME}/config-data/conf {{- if .Values.sts.useHostNetwork }} FQDN="0.0.0.0" @@ -196,11 +141,11 @@ spec: cat "${NIFI_HOME}/conf/nifi.temp" > "${NIFI_HOME}/conf/nifi.properties" - if [[ $(grep $(hostname) conf/authorizers.temp) ]]; then +{{- if .Values.auth.ldap.enabled }} cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" - else +{{- else }} cat "${NIFI_HOME}/conf/authorizers.empty" > "${NIFI_HOME}/conf/authorizers.xml" - fi +{{- end }} if ! test -f /opt/nifi/data/flow.xml.gz && test -f /opt/nifi/data/flow.xml; then gzip /opt/nifi/data/flow.xml @@ -210,19 +155,6 @@ spec: prop_replace nifi.cluster.node.address ${FQDN} prop_replace nifi.zookeeper.connect.string ${NIFI_ZOOKEEPER_CONNECT_STRING} prop_replace nifi.web.http.host ${FQDN} -{{- if .Values.auth.oidc.enabled }} -{{- if .Values.properties.clusterSecure }} - # Update nifi.properties for security properties - prop_replace nifi.web.https.host ${FQDN} - prop_replace nifi.security.keystoreType jks - prop_replace nifi.security.keystore ${NIFI_HOME}/config-data/certs/keystore.jks - prop_replace nifi.security.keystorePasswd $(jq -r .keyStorePassword ${NIFI_HOME}/config-data/certs/config.json) - prop_replace nifi.security.keyPasswd $(jq -r .keyPassword ${NIFI_HOME}/config-data/certs/config.json) - prop_replace nifi.security.truststoreType jks - prop_replace nifi.security.truststore ${NIFI_HOME}/config-data/certs/truststore.jks - prop_replace nifi.security.truststorePasswd $(jq -r .trustStorePassword ${NIFI_HOME}/config-data/certs/config.json) -{{- end }} -{{- end }} {{- if .Values.properties.webProxyHost }} # Update nifi.properties for web ui proxy hostname @@ -243,7 +175,6 @@ spec: function offloadNode() { FQDN=$(hostname -f) echo "disconnecting node '$FQDN'" - {{- if .Values.properties.clusterSecure }} baseUrl=https://${FQDN}:{{ .Values.properties.httpsPort }} keystore=${NIFI_HOME}/config-data/certs/keystore.jks @@ -253,9 +184,6 @@ spec: truststorePasswd=$(jq -r .trustStorePassword ${NIFI_HOME}/config-data/certs/config.json) secureArgs=" --truststore ${truststore} --truststoreType JKS --truststorePasswd ${truststorePasswd} --keystore ${keystore} --keystoreType JKS --keystorePasswd ${keystorePasswd} --proxiedEntity "{{ .Values.auth.admin }}"" - {{- else }} - baseUrl=http://${FQDN}:{{ .Values.properties.httpPort }} - {{- end }} echo baseUrl ${baseUrl} echo "gracefully disconnecting node '$FQDN' from cluster" @@ -266,11 +194,7 @@ spec: echo "" echo "get a connected node" connectedNode=$(jq -r 'first(.cluster.nodes|=sort_by(.address)| .cluster.nodes[] | select(.status=="CONNECTED")) | .address' nodes.json) - {{- if .Values.properties.clusterSecure }} baseUrl=https://${connectedNode}:{{ .Values.properties.httpsPort }} - {{- else }} - baseUrl=http://${connectedNode}:{{ .Values.properties.httpPort }} - {{- end }} echo baseUrl ${baseUrl} echo "" echo "wait until node has state 'DISCONNECTED'" @@ -305,10 +229,11 @@ spec: echo NiFi running with PID ${nifi_pid}. wait ${nifi_pid} - {{ if .Values.auth.ldap }} + {{- if .Values.auth.ldap.enabled }} /opt/nifi/nifi-toolkit-current/bin/tls-toolkit.sh standalone -n '{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local' -C '{{.Values.auth.ldap.admin}}' -o '/opt/nifi/nifi-current/conf/' -P {{.Values.auth.SSL.truststorePasswd}} -S {{.Values.auth.SSL.keystorePasswd}} --nifiPropertiesFile /opt/nifi/nifi-current/conf/nifi.properties exec bin/nifi.sh run - {{ end }} + {{- end }} +{{- end }} resources: {{ toYaml .Values.resources | indent 10 }} ports: @@ -317,32 +242,34 @@ spec: name: metrics protocol: TCP {{- end }} -{{- if .Values.properties.httpsPort }} - containerPort: {{ .Values.properties.httpsPort }} {{- if .Values.sts.hostPort }} hostPort: {{ .Values.sts.hostPort }} {{- end }} name: https protocol: TCP -{{- else }} - - containerPort: {{ .Values.properties.httpPort }} - name: http - protocol: TCP -{{- end }} - containerPort: {{ .Values.properties.clusterPort }} name: cluster protocol: TCP env: - name: NIFI_ZOOKEEPER_CONNECT_STRING value: {{ template "zookeeper.url" . }} +{{- if not (or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled)) }} + - name: SINGLE_USER_CREDENTIALS_USERNAME + value: {{ .Values.auth.singleUser.username }} + - name: SINGLE_USER_CREDENTIALS_PASSWORD + value: {{ .Values.auth.singleUser.password }} + - name: NIFI_WEB_HTTPS_HOST + value: 0.0.0.0 +{{- end }} {{- if .Values.env }} {{ toYaml .Values.env | indent 8 }} {{- end }} -{{- if .Values.postStart }} {{- if .Values.envFrom }} envFrom: {{ toYaml .Values.envFrom | indent 8 }} {{- end }} +{{- if .Values.postStart }} lifecycle: postStart: exec: @@ -353,11 +280,7 @@ spec: initialDelaySeconds: 60 periodSeconds: 20 tcpSocket: -{{- if .Values.properties.httpsPort }} port: {{ .Values.properties.httpsPort }} -{{- else }} - port: {{ .Values.properties.httpPort }} -{{- end }} # exec: # command: # - bash @@ -379,33 +302,28 @@ spec: # jq . $NIFI_BASE_DIR/data/cluster.state # exit 1 # fi -# {{- end }} +{{- end }} livenessProbe: initialDelaySeconds: 90 periodSeconds: 60 tcpSocket: -{{- if .Values.properties.httpsPort }} port: {{ .Values.properties.httpsPort }} -{{- else }} - port: {{ .Values.properties.httpPort }} -{{- end }} volumeMounts: + - name: "logs" + mountPath: /opt/nifi/nifi-current/logs +{{- if or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled) }} - name: "data" mountPath: /opt/nifi/data - name: "auth-conf" mountPath: /opt/nifi/nifi-current/auth-conf/ -{{- if .Values.properties.clusterSecure }} - name: "config-data" mountPath: /opt/nifi/nifi-current/config-data -{{- end }} - name: "flowfile-repository" mountPath: /opt/nifi/flowfile_repository - name: "content-repository" mountPath: /opt/nifi/content_repository - name: "provenance-repository" mountPath: /opt/nifi/provenance_repository - - name: "logs" - mountPath: /opt/nifi/nifi-current/logs - name: "bootstrap-conf" mountPath: /opt/nifi/nifi-current/conf/bootstrap.conf subPath: "bootstrap.conf" @@ -468,6 +386,7 @@ spec: {{- if .Values.extraVolumeMounts }} {{ toYaml .Values.extraVolumeMounts | indent 10 }} {{- end }} +{{- end }} - name: app-log imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" @@ -496,6 +415,7 @@ spec: - name: logs mountPath: /var/log volumes: +{{- if or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled) }} - name: "bootstrap-conf" configMap: name: {{ template "apache-nifi.fullname" . }}-config @@ -560,6 +480,7 @@ spec: configMap: name: {{ .name }} {{- end }} +{{- end }} {{- if not .Values.persistence.enabled }} - name: config-data emptyDir: {} @@ -581,7 +502,17 @@ spec: {{- end }} {{- if .Values.persistence.enabled }} volumeClaimTemplates: -{{- if .Values.properties.clusterSecure }} + - metadata: + name: logs + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.logStorage.size }} - metadata: name: "config-data" spec: @@ -590,7 +521,6 @@ spec: resources: requests: storage: {{ .Values.persistence.configStorage.size }} -{{- end }} - metadata: name: data spec: @@ -635,17 +565,6 @@ spec: resources: requests: storage: {{ .Values.persistence.provenanceRepoStorage.size }} - - metadata: - name: logs - spec: - accessModes: - {{- range .Values.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - storageClassName: {{ .Values.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.persistence.logStorage.size }} - metadata: name: auth-conf spec: @@ -657,4 +576,4 @@ spec: resources: requests: storage: {{ .Values.persistence.authconfStorage.size }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/values.yaml b/values.yaml index 38b63b71..a175e39c 100644 --- a/values.yaml +++ b/values.yaml @@ -7,8 +7,8 @@ replicaCount: 1 ## image: repository: apache/nifi - tag: "1.12.1" - pullPolicy: IfNotPresent + tag: "1.14.0" + pullPolicy: "IfNotPresent" ## Optionally specify an imagePullSecret. ## Secret must be manually created in the namespace. @@ -69,18 +69,16 @@ sts: properties: # use externalSecure for when inbound SSL is provided by nginx-ingress or other external mechanism + sensitiveKey: changeMechangeMe # Must to have minnimal 12 length key + alogorithm: NIFI_PBKDF2_AES_GCM_256 externalSecure: false - isNode: true # set to false if ldap is enabled - httpPort: 8080 # set to null if ldap is enabled - httpsPort: null # set to 9443 if ldap is enabled - webProxyHost: + isNode: false + httpsPort: 8443 + webProxyHost: # : (If Nifi service is NodePort or LoadBalancer) clusterPort: 6007 - clusterSecure: false # set to true if ldap is enabled - needClientAuth: false provenanceStorage: "8 GB" siteToSite: port: 10000 - authorizer: managed-authorizer # use properties.safetyValve to pass explicit 'key: value' pairs that overwrite other configuration safetyValve: #nifi.variable.registry.properties: "${NIFI_HOME}/example1.properties, ${NIFI_HOME}/example2.properties" @@ -99,14 +97,20 @@ properties: auth: admin: CN=admin, OU=NIFI SSL: - keystorePasswd: env:PASS - truststorePasswd: env:PASS + keystorePasswd: changeMe + truststorePasswd: changeMe + + # Automaticaly disabled if OIDC or LDAP enabled + singleUser: + username: username + password: changemechangeme # Must to have at least 12 characters + ldap: enabled: false - host: ldap://: - searchBase: CN=Users,DC=example,DC=com - admin: cn=admin,dc=example,dc=be - pass: password + host: #ldap://: + searchBase: #CN=Users,DC=ldap,DC=example,DC=be + admin: #cn=admin,dc=ldap,dc=example,dc=be + pass: #ChangeMe searchFilter: (objectClass=*) userIdentityAttribute: cn authStrategy: SIMPLE # How the connection to the LDAP server is authenticated. Possible values are ANONYMOUS, SIMPLE, LDAPS, or START_TLS. @@ -115,13 +119,29 @@ auth: oidc: enabled: false - discoveryUrl: - clientId: - clientSecret: - claimIdentifyingUser: email + discoveryUrl: #http://:/auth/realms//.well-known/openid-configuration + clientId: # + clientSecret: # + claimIdentifyingUser: preferred_username ## Request additional scopes, for example profile additionalScopes: +openldap: + enabled: false + persistence: + enabled: true + env: + LDAP_ORGANISATION: # name of your organization e.g. "Example" + LDAP_DOMAIN: # your domain e.g. "ldap.example.be" + LDAP_BACKEND: "hdb" + LDAP_TLS: "true" + LDAP_TLS_ENFORCE: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "false" + adminPassword: #ChengeMe + configPassword: #ChangeMe + customLdifFiles: + 1-default-users.ldif: |- + # You can find an example ldif file at https://github.com/cetic/fadi/blob/master/examples/basic/example.ldif ## Expose the nifi service to be accessed from outside the cluster (LoadBalancer service). ## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. ## ref: http://kubernetes.io/docs/user-guide/services/ @@ -136,8 +156,8 @@ headless: # ui service service: type: ClusterIP - httpPort: 8080 - httpsPort: 9443 + httpsPort: 8443 + # nodePort: 30236 annotations: {} # loadBalancerIP: ## Load Balancer sources