diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56f1dae --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# binary +/bin/* + diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 65347a6..796fca4 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -1,24 +1,65 @@ ##@ App +OS ?= $(shell go env GOOS 2>/dev/null || echo linux) +ARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64) +KUSTOMIZE := ./bin/kustomize +KUSTOMIZE_VERSION ?= v4.5.7 +YQ = ./bin/yq +YQ_VERSION := 4.31.2 + .PHONY: all -all: update-cpi-chart update-csi-chart update-kubevip-chart update-kubevip-cloud-provider-chart +all: update-cpi-chart update-csi-chart apply-custom-patches-for-csi update-kubevip-chart update-kubevip-cloud-provider-chart + @$(call say,Sync has been done ✓) .PHONY: update-cpi-chart -update-cpi-chart: +update-cpi-chart: + @$(call say,CPI helm chart) ./hack/update-cpi-chart.sh ./hack/common-labels-injector.sh cloud-provider-for-vsphere -.PHONY: update-csi-chart -update-csi-chart: - ./hack/update-csi-chart.sh +.PHONY: apply-custom-patches-for-csi +apply-custom-patches-for-csi: $(YQ) ## apply giantswarm specific patches that are not possible via kustomize + @$(call say,Custom yq magic for CSI) + ./hack/custom-patches.sh ./hack/common-labels-injector.sh vsphere-csi-driver +.PHONY: update-csi-chart +update-csi-chart: $(KUSTOMIZE) + @$(call say,CSI helm chart) + ./hack/update-csi-chart.sh $(KUSTOMIZE) + .PHONY: update-kubevip-chart -update-kubevip-chart: +update-kubevip-chart: + @$(call say,Kubevip helm chart) ./hack/update-kubevip-chart.sh ./hack/common-labels-injector.sh kube-vip .PHONY: update-kubevip-cloud-provider-chart -update-kubevip-cloud-provider-chart: +update-kubevip-cloud-provider-chart: + @$(call say,Kubevip cloud provider helm chart) ./hack/update-kubevip-cloud-provider-chart.sh ./hack/common-labels-injector.sh kube-vip-cloud-provider + +$(KUSTOMIZE): ## Download kustomize locally if necessary. + @$(call say,Download Kustomize) + mkdir -p $(dir $@) + curl -sfL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_$(OS)_$(ARCH).tar.gz" | tar zxv -C $(dir $@) + chmod +x $@ + @echo "kustomize downloaded" + +$(YQ): ## Download yq locally if necessary. + @$(call say,Download yq) + curl -sfL https://github.com/mikefarah/yq/releases/download/v$(YQ_VERSION)/yq_$(OS)_$(ARCH) > $@ + chmod +x $@ + @echo "yq downloaded" + + +ifndef NO_COLOR +YELLOW=\033[0;33m +# no color +NC=\033[0m +endif + +define say +echo "\n$(shell echo "$1 " | tr '[:rune:]' '=')\n $(YELLOW)$1$(NC)\n$(shell echo "$1 " | tr '[:rune:]' '=')" +endef diff --git a/config/vsphere-csi-driver/kustomization.yaml b/config/vsphere-csi-driver/kustomization.yaml index 13db989..4f242cd 100644 --- a/config/vsphere-csi-driver/kustomization.yaml +++ b/config/vsphere-csi-driver/kustomization.yaml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization namespace: "{{ .Release.Namespace }}" patchesStrategicMerge: diff --git a/config/vsphere-csi-driver/overwrites/values.yaml b/config/vsphere-csi-driver/overwrites/values.yaml index c02ea43..e8b1a37 100644 --- a/config/vsphere-csi-driver/overwrites/values.yaml +++ b/config/vsphere-csi-driver/overwrites/values.yaml @@ -53,3 +53,9 @@ storageClass: isDefault: false vcdStorageProfileName: "vSAN Default Storage Policy" fileSystem: "ext4" + +containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/hack/custom-patches.sh b/hack/custom-patches.sh new file mode 100755 index 0000000..22c8a22 --- /dev/null +++ b/hack/custom-patches.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +YQ="./bin/yq" + +f="helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml" +${YQ} e '.spec.template.spec.securityContext.remove-this-key="'" +{{- with .Values.podSecurityContext }} + {{- . | toYaml | nindent 8 }} +{{- end }} +"'" | .spec.template.spec.containers[].securityContext.remove-this-key="'" +{{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} +{{- end }} +"'"' ${f} > ${f}.tmp +cat ${f}.tmp | grep -v 'remove-this-key' > ${f} +rm -rf ${f}.tmp diff --git a/hack/update-csi-chart.sh b/hack/update-csi-chart.sh index 0d3a824..72a306d 100755 --- a/hack/update-csi-chart.sh +++ b/hack/update-csi-chart.sh @@ -4,6 +4,7 @@ set -euo pipefail base_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) chart_dir="./helm/cloud-provider-vsphere/charts/vsphere-csi-driver" +KUSTOMIZE="${1:-kustomize}" cd "$base_dir" @@ -24,7 +25,7 @@ cp -R \ # Customizations -kubectl kustomize "./config/vsphere-csi-driver" -o "./config/vsphere-csi-driver/tmp" +${KUSTOMIZE} build "./config/vsphere-csi-driver" -o "./config/vsphere-csi-driver/tmp" find \ "./config/vsphere-csi-driver/tmp/" \ diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml index f3f1668..ec7966e 100644 --- a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml +++ b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/apps_v1_deployment_vsphere-csi-controller.yaml @@ -24,178 +24,210 @@ spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vsphere-csi-controller - topologyKey: kubernetes.io/hostname + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - - args: - - --v=4 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --leader-election - - --kube-api-qps=100 - - --kube-api-burst=100 - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiAttacher.image}}:{{ .Values.controllerDeployment.csiAttacher.tag}}' - name: csi-attacher - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --timeout=300s - - --handle-volume-inuse-error=false - - --csi-address=$(ADDRESS) - - --kube-api-qps=100 - - --kube-api-burst=100 - - --leader-election - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiResizer.image}}:{{ .Values.controllerDeployment.csiResizer.tag}}' - name: csi-resizer - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --fss-name=internal-feature-states.csi.vsphere.vmware.com - - --fss-namespace=$(CSI_NAMESPACE) - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: X_CSI_MODE - value: controller - - name: X_CSI_SPEC_DISABLE_LEN_CHECK - value: "true" - - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT - value: 3m - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf - - name: LOGGER_LEVEL - value: PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "100" - - name: INCLUSTER_CLIENT_BURST - value: "100" - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: '{{ .Values.controllerDeployment.csiController.image}}:{{ .Values.controllerDeployment.csiController.tag}}' - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - name: vsphere-csi-controller - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - - containerPort: 2112 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --csi-address=/csi/csi.sock - image: '{{ .Values.controllerDeployment.livenessProbe.image}}:{{ .Values.controllerDeployment.livenessProbe.tag}}' - name: liveness-probe - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --leader-election - - --fss-name=internal-feature-states.csi.vsphere.vmware.com - - --fss-namespace=$(CSI_NAMESPACE) - env: - - name: FULL_SYNC_INTERVAL_MINUTES - value: "30" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf - - name: LOGGER_LEVEL - value: PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "100" - - name: INCLUSTER_CLIENT_BURST - value: "100" - - name: GODEBUG - value: x509sha1=1 - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: '{{ .Values.controllerDeployment.syncer.image}}:{{ .Values.controllerDeployment.syncer.tag}}' - imagePullPolicy: Always - name: vsphere-syncer - ports: - - containerPort: 2113 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - - args: - - --v=4 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --kube-api-qps=100 - - --kube-api-burst=100 - - --leader-election - - --default-fstype=ext4 - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiProvisioner.image}}:{{ .Values.controllerDeployment.csiProvisioner.tag}}' - name: csi-provisioner - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=4 - - --kube-api-qps=100 - - --kube-api-burst=100 - - --timeout=300s - - --csi-address=$(ADDRESS) - - --leader-election - env: - - name: ADDRESS - value: /csi/csi.sock - image: '{{ .Values.controllerDeployment.csiSnapshotter.image}}:{{ .Values.controllerDeployment.csiSnapshotter.tag}}' - name: csi-snapshotter - volumeMounts: - - mountPath: /csi - name: socket-dir + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --kube-api-qps=100 + - --kube-api-burst=100 + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiAttacher.image}}:{{ .Values.controllerDeployment.csiAttacher.tag}}' + name: csi-attacher + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiResizer.image}}:{{ .Values.controllerDeployment.csiResizer.tag}}' + name: csi-resizer + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ .Values.controllerDeployment.csiController.image}}:{{ .Values.controllerDeployment.csiController.tag}}' + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + name: vsphere-csi-controller + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: '{{ .Values.controllerDeployment.livenessProbe.image}}:{{ .Values.controllerDeployment.livenessProbe.tag}}' + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --leader-election + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ .Values.controllerDeployment.syncer.image}}:{{ .Values.controllerDeployment.syncer.tag}}' + imagePullPolicy: Always + name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiProvisioner.image}}:{{ .Values.controllerDeployment.csiProvisioner.tag}}' + name: csi-provisioner + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + env: + - name: ADDRESS + value: /csi/csi.sock + image: '{{ .Values.controllerDeployment.csiSnapshotter.image}}:{{ .Values.controllerDeployment.csiSnapshotter.tag}}' + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + securityContext: + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 12 }} + {{- end }} dnsPolicy: Default nodeSelector: node-role.kubernetes.io/control-plane: "" serviceAccountName: vsphere-csi-controller tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: vsphere-config-secret - - emptyDir: {} - name: socket-dir + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - emptyDir: {} + name: socket-dir + securityContext: + {{- with .Values.podSecurityContext }} + {{- . | toYaml | nindent 8 }} + {{- end }} diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml index c02ea43..e8b1a37 100644 --- a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml +++ b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/values.yaml @@ -53,3 +53,9 @@ storageClass: isDefault: false vcdStorageProfileName: "vSAN Default Storage Policy" fileSystem: "ext4" + +containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/helm/cloud-provider-vsphere/values.schema.json b/helm/cloud-provider-vsphere/values.schema.json index 33b33b4..4251986 100644 --- a/helm/cloud-provider-vsphere/values.schema.json +++ b/helm/cloud-provider-vsphere/values.schema.json @@ -15,6 +15,28 @@ } } }, + "crdInstall": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "kubectl": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "registry": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + } + } + }, "global": { "type": "object", "properties": { @@ -49,6 +71,14 @@ "type": "string" } } + }, + "podSecurityStandards": { + "type": "object", + "properties": { + "enforced": { + "type": "boolean" + } + } } } }, diff --git a/helm/cloud-provider-vsphere/values.yaml b/helm/cloud-provider-vsphere/values.yaml index 3c964c2..e7850b8 100644 --- a/helm/cloud-provider-vsphere/values.yaml +++ b/helm/cloud-provider-vsphere/values.yaml @@ -11,6 +11,9 @@ global: # openssl s_client -connect :443 < /dev/null 2>/dev/null | openssl x509 -fingerprint -sha1 -noout thumbprint: "" + podSecurityStandards: + enforced: false + cloud-provider-for-vsphere: daemonset: image: docker.io/giantswarm/cpi-vsphere-manager @@ -22,6 +25,7 @@ kube-vip: image: repository: docker.io/giantswarm/kube-vip tag: "v0.5.11" + nameOverride: kube-vip-svc-lb tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane @@ -37,3 +41,33 @@ kube-vip-cloud-provider: image: repository: docker.io/giantswarm/kube-vip-cloud-provider tag: "v0.0.4" + nameOverride: kube-vip-cloud-provider + +vsphere-csi-driver: + containerSecurityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + runAsNonRoot: false + seccompProfile: + type: RuntimeDefault + + podSecurityContext: + runAsNonRoot: false + seccompProfile: + type: RuntimeDefault + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault