From 5f0494f3d708a2b839c197ce9fc92ad7e1d7c3af Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Thu, 5 Oct 2023 15:20:42 +0200 Subject: [PATCH] Use kustomize for csi driver Signed-off-by: Jirka Kremser --- .gitignore | 2 +- Makefile.custom.mk | 18 +- .../overwrites/templates/pss-exceptions.yaml | 65 ---- .../overwrites/templates/pss-exceptions.yaml | 65 ---- .../vsphere-csi-driver/overwrites/values.yaml | 6 + hack/custom-patches.sh | 20 + .../templates/daemonset.yaml | 3 + .../templates/pss-exceptions.yaml | 34 -- .../templates/pss-exceptions.yaml | 34 -- ..._v1_deployment_vsphere-csi-controller.yaml | 368 ++++++++++-------- .../charts/vsphere-csi-driver/values.yaml | 6 + .../policyexceptions-crd.yaml} | 0 ...d-provider-for-vsphere-pss-exceptions.yaml | 4 +- ...ube-vip-cloud-provider-pss-exceptions.yaml | 2 +- .../kube-vip-pss-exceptions.yaml} | 16 +- .../vsphere-csi-driver-pss-exceptions.yaml} | 30 +- .../templates/_helpers.tpl | 4 +- .../templates/crd-install/cr-configmap.yaml | 27 ++ .../templates/crd-install/crd-configmap.yaml | 2 +- .../templates/crd-install/crd-job.yaml | 74 +++- .../templates/crd-install/crd-netpol.yaml | 2 +- .../templates/crd-install/crd-rbac.yaml | 10 +- .../crd-install/crd-serviceaccount.yaml | 2 +- helm/cloud-provider-vsphere/values.yaml | 2 + 24 files changed, 375 insertions(+), 421 deletions(-) delete mode 100644 config/kube-vip/overwrites/templates/pss-exceptions.yaml delete mode 100644 config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml create mode 100755 hack/custom-patches.sh delete mode 100644 helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml delete mode 100644 helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml rename helm/cloud-provider-vsphere/{templates/files/policyexceptions.yaml => files/policyexceptions-crd.yaml} (100%) rename config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml => helm/cloud-provider-vsphere/polex/cloud-provider-for-vsphere-pss-exceptions.yaml (93%) rename config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml => helm/cloud-provider-vsphere/polex/kube-vip-cloud-provider-pss-exceptions.yaml (98%) rename helm/cloud-provider-vsphere/{charts/kube-vip/templates/pss-exceptions.yaml => polex/kube-vip-pss-exceptions.yaml} (78%) rename helm/cloud-provider-vsphere/{charts/vsphere-csi-driver/templates/pss-exceptions.yaml => polex/vsphere-csi-driver-pss-exceptions.yaml} (73%) create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/cr-configmap.yaml diff --git a/.gitignore b/.gitignore index 839d19c..56f1dae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ # binary -/bin/kustomize +/bin/* diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 93ce4cd..796fca4 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -4,9 +4,11 @@ OS ?= $(shell go env GOOS 2>/dev/null || echo linux) ARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64) KUSTOMIZE := ./bin/kustomize KUSTOMIZE_VERSION ?= v4.5.7 +YQ = ./bin/yq +YQ_VERSION := 4.31.2 .PHONY: all -all: update-cpi-chart update-csi-chart update-kubevip-chart update-kubevip-cloud-provider-chart +all: update-cpi-chart update-csi-chart apply-custom-patches-for-csi update-kubevip-chart update-kubevip-cloud-provider-chart @$(call say,Sync has been done ✓) .PHONY: update-cpi-chart @@ -15,11 +17,16 @@ update-cpi-chart: ./hack/update-cpi-chart.sh ./hack/common-labels-injector.sh cloud-provider-for-vsphere +.PHONY: apply-custom-patches-for-csi +apply-custom-patches-for-csi: $(YQ) ## apply giantswarm specific patches that are not possible via kustomize + @$(call say,Custom yq magic for CSI) + ./hack/custom-patches.sh + ./hack/common-labels-injector.sh vsphere-csi-driver + .PHONY: update-csi-chart update-csi-chart: $(KUSTOMIZE) @$(call say,CSI helm chart) ./hack/update-csi-chart.sh $(KUSTOMIZE) - ./hack/common-labels-injector.sh vsphere-csi-driver .PHONY: update-kubevip-chart update-kubevip-chart: @@ -40,6 +47,13 @@ $(KUSTOMIZE): ## Download kustomize locally if necessary. chmod +x $@ @echo "kustomize downloaded" +$(YQ): ## Download yq locally if necessary. + @$(call say,Download yq) + curl -sfL https://github.com/mikefarah/yq/releases/download/v$(YQ_VERSION)/yq_$(OS)_$(ARCH) > $@ + chmod +x $@ + @echo "yq downloaded" + + ifndef NO_COLOR YELLOW=\033[0;33m # no color diff --git a/config/kube-vip/overwrites/templates/pss-exceptions.yaml b/config/kube-vip/overwrites/templates/pss-exceptions.yaml deleted file mode 100644 index e766875..0000000 --- a/config/kube-vip/overwrites/templates/pss-exceptions.yaml +++ /dev/null @@ -1,65 +0,0 @@ -{{- if .Values.global.kyvernoPolicyExceptions.enabled }} -apiVersion: kyverno.io/v2alpha1 -kind: PolicyException -metadata: - name: kube-vip-exceptions - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels.common" . | nindent 4 }} -spec: - exceptions: - - policyName: disallow-capabilities - ruleNames: - - autogen-adding-capabilities - - adding-capabilities - - policyName: disallow-capabilities-strict - ruleNames: - - autogen-require-drop-all - - require-drop-all - - autogen-adding-capabilities-strict - - adding-capabilities-strict - - policyName: disallow-host-namespaces - ruleNames: - - autogen-host-namespaces - - host-namespaces - - policyName: disallow-privilege-escalation - ruleNames: - - autogen-privilege-escalation - - privilege-escalation - - policyName: require-run-as-nonroot - ruleNames: - - autogen-run-as-non-root - - run-as-non-root - - policyName: restrict-seccomp-strict - ruleNames: - - autogen-check-seccomp-strict - - check-seccomp-strict - - policyName: restrict-image-registries - ruleNames: - - autogen-validate-registries - - validate-registries - - policyName: restrict-volume-types - ruleNames: - - autogen-restricted-volumes - - restricted-volumes - match: - any: - - resources: - kinds: - - Pod - # because kube-vip prefix is common also for kube-vip-cloud-provider, we need to distinguish bethween the two - selector: - matchLabels: - {{- include "kube-vip.selectorLabels" . | nindent 10 }} - namespaces: - - {{ .Release.Namespace }} - names: - - kube-vip-* - - resources: - kinds: - - DaemonSet - namespaces: - - {{ .Release.Namespace }} - names: - - kube-vip -{{- end }} \ No newline at end of file diff --git a/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml b/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml deleted file mode 100644 index 1e7f41b..0000000 --- a/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml +++ /dev/null @@ -1,65 +0,0 @@ -{{- if .Values.global.kyvernoPolicyExceptions.enabled }} -apiVersion: kyverno.io/v2alpha1 -kind: PolicyException -metadata: - name: vsphere-csi-exceptions - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels.common" . | nindent 4 }} -spec: - exceptions: - - policyName: disallow-privileged-containers - ruleNames: - - autogen-privileged-containers - - privileged-containers - - policyName: disallow-host-path - ruleNames: - - autogen-host-path - - host-path - - policyName: disallow-host-namespaces - ruleNames: - - host-namespaces - - autogen-host-namespaces - - policyName: disallow-host-ports - ruleNames: - - host-ports-none - - autogen-host-ports-none - - policyName: restrict-volume-types - ruleNames: - - restricted-volumes - - autogen-restricted-volumes - - policyName: require-run-as-nonroot - ruleNames: - - run-as-non-root - - autogen-run-as-non-root - - policyName: restrict-seccomp-strict - ruleNames: - - check-seccomp-strict - - autogen-check-seccomp-strict - - policyName: disallow-capabilities - ruleNames: - - adding-capabilities - - autogen-adding-capabilities - - policyName: disallow-capabilities-strict - ruleNames: - - adding-capabilities-strict - - autogen-adding-capabilities-strict - - require-drop-all - - autogen-require-drop-all - - policyName: disallow-privilege-escalation - ruleNames: - - privilege-escalation - - autogen-privilege-escalation - match: - any: - - resources: - kinds: - - DaemonSet - - Deployment - - ReplicaSet - - Pod - namespaces: - - {{ .Release.Namespace }} - names: - - vsphere-csi* -{{- end }} \ No newline at end of file diff --git a/config/vsphere-csi-driver/overwrites/values.yaml b/config/vsphere-csi-driver/overwrites/values.yaml index c02ea43..e8b1a37 100644 --- a/config/vsphere-csi-driver/overwrites/values.yaml +++ b/config/vsphere-csi-driver/overwrites/values.yaml @@ -53,3 +53,9 @@ storageClass: isDefault: false vcdStorageProfileName: "vSAN Default Storage Policy" fileSystem: "ext4" + +containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/hack/custom-patches.sh b/hack/custom-patches.sh new file mode 100755 index 0000000..22c8a22 --- /dev/null +++ b/hack/custom-patches.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +YQ="./bin/yq" + +f="helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml" +${YQ} e '.spec.template.spec.securityContext.remove-this-key="'" +{{- with .Values.podSecurityContext }} + {{- . | toYaml | nindent 8 }} +{{- end }} +"'" | .spec.template.spec.containers[].securityContext.remove-this-key="'" +{{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} +{{- end }} +"'"' ${f} > ${f}.tmp +cat ${f}.tmp | grep -v 'remove-this-key' > ${f} +rm -rf ${f}.tmp diff --git a/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/daemonset.yaml b/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/daemonset.yaml index e888742..3c76d50 100644 --- a/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/daemonset.yaml +++ b/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/daemonset.yaml @@ -45,6 +45,9 @@ spec: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule + - key: node.cluster.x-k8s.io/uninitialized + effect: NoSchedule + operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule operator: Exists diff --git a/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml deleted file mode 100644 index 197a3fe..0000000 --- a/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.global.kyvernoPolicyExceptions.enabled }} -apiVersion: kyverno.io/v2alpha1 -kind: PolicyException -metadata: - name: cloud-provider-exceptions - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels.common" . | nindent 4 }} -spec: - exceptions: - - policyName: disallow-host-path - ruleNames: - - autogen-host-path - - host-path - - policyName: disallow-host-namespaces - ruleNames: - - host-namespaces - - autogen-host-namespaces - - policyName: restrict-seccomp-strict - ruleNames: - - check-seccomp-strict - - autogen-check-seccomp-strict - match: - any: - - resources: - kinds: - - DaemonSet - - ReplicaSet - - Pod - namespaces: - - {{ .Release.Namespace }} - names: - - cloud-provider-for-vsphere* -{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml deleted file mode 100644 index 7e105f1..0000000 --- a/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.global.kyvernoPolicyExceptions.enabled }} -apiVersion: kyverno.io/v2alpha1 -kind: PolicyException -metadata: - name: kube-vip-cloud-provider-exceptions - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels.common" . | nindent 4 }} -spec: - exceptions: - - policyName: disallow-host-path - ruleNames: - - autogen-host-path - - host-path - - policyName: disallow-host-namespaces - ruleNames: - - host-namespaces - - autogen-host-namespaces - - policyName: restrict-seccomp-strict - ruleNames: - - check-seccomp-strict - - autogen-check-seccomp-strict - match: - any: - - resources: - kinds: - - Deployment - - ReplicaSet - - Pod - namespaces: - - {{ .Release.Namespace }} - names: - - kube-vip-cloud-provider* -{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml index f3f1668..ec7966e 100644 --- a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml +++ b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml @@ -24,178 +24,210 @@ spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vsphere-csi-controller - topologyKey: kubernetes.io/hostname + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - - args: - - --v=4 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --leader-election - - --kube-api-qps=100 - - --kube-api-burst=100 - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiAttacher.image}}:{{ .Values.controllerDeployment.csiAttacher.tag}}' - name: csi-attacher - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --timeout=300s - - --handle-volume-inuse-error=false - - --csi-address=$(ADDRESS) - - --kube-api-qps=100 - - --kube-api-burst=100 - - --leader-election - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiResizer.image}}:{{ .Values.controllerDeployment.csiResizer.tag}}' - name: csi-resizer - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --fss-name=internal-feature-states.csi.vsphere.vmware.com - - --fss-namespace=$(CSI_NAMESPACE) - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: X_CSI_MODE - value: controller - - name: X_CSI_SPEC_DISABLE_LEN_CHECK - value: "true" - - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT - value: 3m - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf - - name: LOGGER_LEVEL - value: PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "100" - - name: INCLUSTER_CLIENT_BURST - value: "100" - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: '{{ .Values.controllerDeployment.csiController.image}}:{{ .Values.controllerDeployment.csiController.tag}}' - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - name: vsphere-csi-controller - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - - containerPort: 2112 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --csi-address=/csi/csi.sock - image: '{{ .Values.controllerDeployment.livenessProbe.image}}:{{ .Values.controllerDeployment.livenessProbe.tag}}' - name: liveness-probe - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --leader-election - - --fss-name=internal-feature-states.csi.vsphere.vmware.com - - --fss-namespace=$(CSI_NAMESPACE) - env: - - name: FULL_SYNC_INTERVAL_MINUTES - value: "30" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf - - name: LOGGER_LEVEL - value: PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "100" - - name: INCLUSTER_CLIENT_BURST - value: "100" - - name: GODEBUG - value: x509sha1=1 - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: '{{ .Values.controllerDeployment.syncer.image}}:{{ .Values.controllerDeployment.syncer.tag}}' - imagePullPolicy: Always - name: vsphere-syncer - ports: - - containerPort: 2113 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - - args: - - --v=4 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --kube-api-qps=100 - - --kube-api-burst=100 - - --leader-election - - --default-fstype=ext4 - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiProvisioner.image}}:{{ .Values.controllerDeployment.csiProvisioner.tag}}' - name: csi-provisioner - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --kube-api-qps=100 - - --kube-api-burst=100 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --leader-election - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiSnapshotter.image}}:{{ .Values.controllerDeployment.csiSnapshotter.tag}}' - name: csi-snapshotter - volumeMounts: - - mountPath: /csi - name: socket-dir + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --kube-api-qps=100 + - --kube-api-burst=100 + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiAttacher.image}}:{{ .Values.controllerDeployment.csiAttacher.tag}}' + name: csi-attacher + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiResizer.image}}:{{ .Values.controllerDeployment.csiResizer.tag}}' + name: csi-resizer + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ .Values.controllerDeployment.csiController.image}}:{{ .Values.controllerDeployment.csiController.tag}}' + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + name: vsphere-csi-controller + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: '{{ .Values.controllerDeployment.livenessProbe.image}}:{{ .Values.controllerDeployment.livenessProbe.tag}}' + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --leader-election + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ .Values.controllerDeployment.syncer.image}}:{{ .Values.controllerDeployment.syncer.tag}}' + imagePullPolicy: Always + name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiProvisioner.image}}:{{ .Values.controllerDeployment.csiProvisioner.tag}}' + name: csi-provisioner + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiSnapshotter.image}}:{{ .Values.controllerDeployment.csiSnapshotter.tag}}' + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} dnsPolicy: Default nodeSelector: node-role.kubernetes.io/control-plane: "" serviceAccountName: vsphere-csi-controller tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: vsphere-config-secret - - emptyDir: {} - name: socket-dir + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - emptyDir: {} + name: socket-dir + securityContext: + {{- with .Values.podSecurityContext }} + {{- . | toYaml | nindent 8 }} + {{- end }} diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml index c02ea43..e8b1a37 100644 --- a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml +++ b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml @@ -53,3 +53,9 @@ storageClass: isDefault: false vcdStorageProfileName: "vSAN Default Storage Policy" fileSystem: "ext4" + +containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml b/helm/cloud-provider-vsphere/files/policyexceptions-crd.yaml similarity index 100% rename from helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml rename to helm/cloud-provider-vsphere/files/policyexceptions-crd.yaml diff --git a/config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/polex/cloud-provider-for-vsphere-pss-exceptions.yaml similarity index 93% rename from config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml rename to helm/cloud-provider-vsphere/polex/cloud-provider-for-vsphere-pss-exceptions.yaml index 197a3fe..8e99be2 100644 --- a/config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml +++ b/helm/cloud-provider-vsphere/polex/cloud-provider-for-vsphere-pss-exceptions.yaml @@ -2,7 +2,7 @@ apiVersion: kyverno.io/v2alpha1 kind: PolicyException metadata: - name: cloud-provider-exceptions + name: vsphere-cloud-provider-exceptions namespace: {{ .Release.Namespace }} labels: {{- include "labels.common" . | nindent 4 }} @@ -31,4 +31,4 @@ spec: - {{ .Release.Namespace }} names: - cloud-provider-for-vsphere* -{{- end }} \ No newline at end of file +{{- end }} diff --git a/config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/polex/kube-vip-cloud-provider-pss-exceptions.yaml similarity index 98% rename from config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml rename to helm/cloud-provider-vsphere/polex/kube-vip-cloud-provider-pss-exceptions.yaml index 7e105f1..1a6ec10 100644 --- a/config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml +++ b/helm/cloud-provider-vsphere/polex/kube-vip-cloud-provider-pss-exceptions.yaml @@ -31,4 +31,4 @@ spec: - {{ .Release.Namespace }} names: - kube-vip-cloud-provider* -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/polex/kube-vip-pss-exceptions.yaml similarity index 78% rename from helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml rename to helm/cloud-provider-vsphere/polex/kube-vip-pss-exceptions.yaml index e766875..0d455d9 100644 --- a/helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml +++ b/helm/cloud-provider-vsphere/polex/kube-vip-pss-exceptions.yaml @@ -44,22 +44,12 @@ spec: - restricted-volumes match: any: - - resources: - kinds: - - Pod - # because kube-vip prefix is common also for kube-vip-cloud-provider, we need to distinguish bethween the two - selector: - matchLabels: - {{- include "kube-vip.selectorLabels" . | nindent 10 }} - namespaces: - - {{ .Release.Namespace }} - names: - - kube-vip-* - resources: kinds: - DaemonSet + - Pod namespaces: - {{ .Release.Namespace }} names: - - kube-vip -{{- end }} \ No newline at end of file + - kube-vip-svc-lb* +{{- end }} diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/polex/vsphere-csi-driver-pss-exceptions.yaml similarity index 73% rename from helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml rename to helm/cloud-provider-vsphere/polex/vsphere-csi-driver-pss-exceptions.yaml index 1e7f41b..554c12b 100644 --- a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml +++ b/helm/cloud-provider-vsphere/polex/vsphere-csi-driver-pss-exceptions.yaml @@ -28,24 +28,24 @@ spec: ruleNames: - restricted-volumes - autogen-restricted-volumes - - policyName: require-run-as-nonroot - ruleNames: - - run-as-non-root - - autogen-run-as-non-root + # - policyName: require-run-as-nonroot + # ruleNames: + # - run-as-non-root + # - autogen-run-as-non-root - policyName: restrict-seccomp-strict ruleNames: - check-seccomp-strict - autogen-check-seccomp-strict - - policyName: disallow-capabilities - ruleNames: - - adding-capabilities - - autogen-adding-capabilities - - policyName: disallow-capabilities-strict - ruleNames: - - adding-capabilities-strict - - autogen-adding-capabilities-strict - - require-drop-all - - autogen-require-drop-all + # - policyName: disallow-capabilities + # ruleNames: + # - adding-capabilities + # - autogen-adding-capabilities + # - policyName: disallow-capabilities-strict + # ruleNames: + # - adding-capabilities-strict + # - autogen-adding-capabilities-strict + # - require-drop-all + # - autogen-require-drop-all - policyName: disallow-privilege-escalation ruleNames: - privilege-escalation @@ -62,4 +62,4 @@ spec: - {{ .Release.Namespace }} names: - vsphere-csi* -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/_helpers.tpl b/helm/cloud-provider-vsphere/templates/_helpers.tpl index 508ff79..945202a 100644 --- a/helm/cloud-provider-vsphere/templates/_helpers.tpl +++ b/helm/cloud-provider-vsphere/templates/_helpers.tpl @@ -32,8 +32,8 @@ CRD installation labels {{- end -}} {{- define "crdInstallAnnotations" -}} -"helm.sh/hook": "pre-install,pre-upgrade" -"helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed" +"helm.sh/hook": "post-install,post-upgrade" +"helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" {{- end -}} {{/* Create a label which can be used to select any orphaned crd-install hook resources */}} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/cr-configmap.yaml b/helm/cloud-provider-vsphere/templates/crd-install/cr-configmap.yaml new file mode 100644 index 0000000..afb9a76 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/cr-configmap.yaml @@ -0,0 +1,27 @@ +{{/* +We have to create individual configmaps for each CR - they exceed the total +allowed length for a configmap if they are combined. +*/}} +{{ $currentScope := . }} +{{- if .Values.global.crdInstall.enabled }} + {{- range $path, $_ := .Files.Glob "polex/**" }} + {{- with $currentScope }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-15" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +data: + content: | +{{- tpl ($.Files.Get $path) . | indent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml index f2c5f86..243a38e 100644 --- a/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml @@ -24,4 +24,4 @@ data: {{ tpl ($.Files.Get $path) . | indent 4 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml index 61d6af5..3a7b0c2 100644 --- a/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml @@ -35,11 +35,35 @@ spec: - key: node.kubernetes.io/not-ready effect: NoSchedule operator: Exists - containers: - - name: kubectl - image: "{{ .Values.crdInstall.kubectl.registry }}/{{ .Values.crdInstall.kubectl.image }}:{{ .Values.crdInstall.kubectl.tag }}" + volumes: +{{ $currentScope := . }} +{{- range $path, $_ := .Files.Glob "files/**" }} + {{- with $currentScope }} + - name: {{ $path | base | trimSuffix ".yaml" }} + configMap: + name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} + items: + - key: content + path: {{ $path | base }} +{{- end }} +{{- end }} +{{ $currentScope := . }} +{{- range $path, $_ := .Files.Glob "polex/**" }} + {{- with $currentScope }} + - name: {{ $path | base | trimSuffix ".yaml" }} + configMap: + name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} + items: + - key: content + path: {{ $path | base }} +{{- end }} +{{- end }} + initContainers: + - name: install-crds + image: "{{ .Values.global.crdInstall.kubectl.registry }}/{{ .Values.global.crdInstall.kubectl.image }}:{{ .Values.global.crdInstall.kubectl.tag }}" command: - sh + args: - -c - | set -o errexit ; set -o xtrace ; set -o nounset @@ -57,19 +81,39 @@ spec: mountPath: /data/{{ $path | base }} subPath: {{ $path | base }} {{- end }} - resources: {{- toYaml .Values.crdInstall.resources | nindent 10 }} - volumes: -{{ $currentScope := . }} -{{- range $path, $_ := .Files.Glob "files/**" }} - {{- with $currentScope }} - - name: {{ $path | base | trimSuffix ".yaml" }} - configMap: - name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} - items: - - key: content - path: {{ $path | base }} -{{- end }} + containers: + - name: create-in-cluster-ip-pool + image: "{{ .Values.global.crdInstall.kubectl.registry }}/{{ .Values.global.crdInstall.kubectl.image }}:{{ .Values.global.crdInstall.kubectl.tag }}" + imagePullPolicy: IfNotPresent + securityContext: + readOnlyRootFilesystem: true + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 10 }} + {{- end }} + volumeMounts: +{{- range $path, $_ := .Files.Glob "polex/**" }} + - name: {{ $path | base | trimSuffix ".yaml" }} + mountPath: /data/{{ $path | base }} + subPath: {{ $path | base }} {{- end }} + command: + - sh + args: + - -c + - | + set -o nounset + for i in $(seq 20) + do + kubectl apply -f /data/ 2>&1 && exit 0 + _sec=$(echo "1.5^$i" | bc) + echo "Waiting ${_sec} seconds.." + sleep ${_sec} + done + exit 1 + {{- with .Values.containerSecurityContext }} + securityContext: + {{- . | toYaml | nindent 10 }} + {{- end }} restartPolicy: Never backoffLimit: 10 {{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml index 97e899a..c375529 100644 --- a/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml @@ -33,4 +33,4 @@ spec: policyTypes: - Egress - Ingress -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml index 5d51797..367eaf7 100644 --- a/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml @@ -28,6 +28,14 @@ rules: - delete - get - patch +- apiGroups: + - kyverno.io + resources: + - policyexceptions + verbs: + - create + - delete + - get {{- if not .Values.global.podSecurityStandards.enforced }} {{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} - apiGroups: @@ -61,4 +69,4 @@ subjects: - kind: ServiceAccount name: {{ include "crdInstall" . }} namespace: {{ .Release.Namespace | quote }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml index 57edf33..bf2caea 100644 --- a/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml @@ -10,6 +10,6 @@ metadata: {{- include "crdInstallAnnotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "crdInstall" . | quote }} - {{- include "labels.selector" . | nindent 4 }} + {{- include "labels.crdSelector" . | nindent 4 }} role: {{ include "crdInstallSelector" . | quote }} {{- end }} diff --git a/helm/cloud-provider-vsphere/values.yaml b/helm/cloud-provider-vsphere/values.yaml index a22b576..7854994 100644 --- a/helm/cloud-provider-vsphere/values.yaml +++ b/helm/cloud-provider-vsphere/values.yaml @@ -39,6 +39,7 @@ kube-vip: image: repository: docker.io/giantswarm/kube-vip tag: "v0.5.11" + nameOverride: kube-vip-svc-lb tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane @@ -54,6 +55,7 @@ kube-vip-cloud-provider: image: repository: docker.io/giantswarm/kube-vip-cloud-provider tag: "v0.0.4" + nameOverride: kube-vip-cloud-provider containerSecurityContext: allowPrivilegeEscalation: false