From 4000124f6e508667d3ad7d56146ba94ed5677904 Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Thu, 5 Oct 2023 14:02:45 +0200 Subject: [PATCH] Whitelist the kyverno policy exceptions Signed-off-by: Jirka Kremser --- .gitignore | 3 + Makefile.custom.mk | 37 +- .../overwrites/templates/pss-exceptions.yaml | 34 ++ .../overwrites/templates/pss-exceptions.yaml | 34 ++ .../overwrites/templates/pss-exceptions.yaml | 65 +++ config/vsphere-csi-driver/kustomization.yaml | 2 + .../overwrites/templates/pss-exceptions.yaml | 65 +++ hack/update-csi-chart.sh | 3 +- .../templates/pss-exceptions.yaml | 34 ++ .../templates/pss-exceptions.yaml | 34 ++ .../kube-vip/templates/pss-exceptions.yaml | 65 +++ .../templates/pss-exceptions.yaml | 65 +++ .../templates/_helpers.tpl | 26 + .../crd-install/crd-ciliumnetworkpolicy.yaml | 23 + .../templates/crd-install/crd-configmap.yaml | 27 + .../templates/crd-install/crd-job.yaml | 75 +++ .../templates/crd-install/crd-netpol.yaml | 36 ++ .../templates/crd-install/crd-psp.yaml | 40 ++ .../templates/crd-install/crd-rbac.yaml | 64 +++ .../crd-install/crd-serviceaccount.yaml | 15 + .../templates/files/policyexceptions.yaml | 523 ++++++++++++++++++ .../cloud-provider-vsphere/values.schema.json | 38 ++ helm/cloud-provider-vsphere/values.yaml | 31 ++ 23 files changed, 1333 insertions(+), 6 deletions(-) create mode 100644 .gitignore create mode 100644 config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml create mode 100644 config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml create mode 100644 config/kube-vip/overwrites/templates/pss-exceptions.yaml create mode 100644 config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml create mode 100644 helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml create mode 100644 helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml create mode 100644 helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml create mode 100644 helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-ciliumnetworkpolicy.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-psp.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml create mode 100644 helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml create mode 100644 helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..839d19c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# binary +/bin/kustomize + diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 65347a6..93ce4cd 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -1,24 +1,51 @@ ##@ App +OS ?= $(shell go env GOOS 2>/dev/null || echo linux) +ARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64) +KUSTOMIZE := ./bin/kustomize +KUSTOMIZE_VERSION ?= v4.5.7 + .PHONY: all all: update-cpi-chart update-csi-chart update-kubevip-chart update-kubevip-cloud-provider-chart + @$(call say,Sync has been done ✓) .PHONY: update-cpi-chart -update-cpi-chart: +update-cpi-chart: + @$(call say,CPI helm chart) ./hack/update-cpi-chart.sh ./hack/common-labels-injector.sh cloud-provider-for-vsphere .PHONY: update-csi-chart -update-csi-chart: - ./hack/update-csi-chart.sh +update-csi-chart: $(KUSTOMIZE) + @$(call say,CSI helm chart) + ./hack/update-csi-chart.sh $(KUSTOMIZE) ./hack/common-labels-injector.sh vsphere-csi-driver .PHONY: update-kubevip-chart -update-kubevip-chart: +update-kubevip-chart: + @$(call say,Kubevip helm chart) ./hack/update-kubevip-chart.sh ./hack/common-labels-injector.sh kube-vip .PHONY: update-kubevip-cloud-provider-chart -update-kubevip-cloud-provider-chart: +update-kubevip-cloud-provider-chart: + @$(call say,Kubevip cloud provider helm chart) ./hack/update-kubevip-cloud-provider-chart.sh ./hack/common-labels-injector.sh kube-vip-cloud-provider + +$(KUSTOMIZE): ## Download kustomize locally if necessary. + @$(call say,Download Kustomize) + mkdir -p $(dir $@) + curl -sfL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_$(OS)_$(ARCH).tar.gz" | tar zxv -C $(dir $@) + chmod +x $@ + @echo "kustomize downloaded" + +ifndef NO_COLOR +YELLOW=\033[0;33m +# no color +NC=\033[0m +endif + +define say +echo "\n$(shell echo "$1 " | tr '[:rune:]' '=')\n $(YELLOW)$1$(NC)\n$(shell echo "$1 " | tr '[:rune:]' '=')" +endef diff --git a/config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml b/config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml new file mode 100644 index 0000000..197a3fe --- /dev/null +++ b/config/cloud-provider-for-vsphere/overwrites/templates/pss-exceptions.yaml @@ -0,0 +1,34 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: cloud-provider-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + match: + any: + - resources: + kinds: + - DaemonSet + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - cloud-provider-for-vsphere* +{{- end }} \ No newline at end of file diff --git a/config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml b/config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml new file mode 100644 index 0000000..7e105f1 --- /dev/null +++ b/config/kube-vip-cloud-provider/overwrites/templates/pss-exceptions.yaml @@ -0,0 +1,34 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: kube-vip-cloud-provider-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + match: + any: + - resources: + kinds: + - Deployment + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip-cloud-provider* +{{- end }} \ No newline at end of file diff --git a/config/kube-vip/overwrites/templates/pss-exceptions.yaml b/config/kube-vip/overwrites/templates/pss-exceptions.yaml new file mode 100644 index 0000000..e766875 --- /dev/null +++ b/config/kube-vip/overwrites/templates/pss-exceptions.yaml @@ -0,0 +1,65 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: kube-vip-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-capabilities + ruleNames: + - autogen-adding-capabilities + - adding-capabilities + - policyName: disallow-capabilities-strict + ruleNames: + - autogen-require-drop-all + - require-drop-all + - autogen-adding-capabilities-strict + - adding-capabilities-strict + - policyName: disallow-host-namespaces + ruleNames: + - autogen-host-namespaces + - host-namespaces + - policyName: disallow-privilege-escalation + ruleNames: + - autogen-privilege-escalation + - privilege-escalation + - policyName: require-run-as-nonroot + ruleNames: + - autogen-run-as-non-root + - run-as-non-root + - policyName: restrict-seccomp-strict + ruleNames: + - autogen-check-seccomp-strict + - check-seccomp-strict + - policyName: restrict-image-registries + ruleNames: + - autogen-validate-registries + - validate-registries + - policyName: restrict-volume-types + ruleNames: + - autogen-restricted-volumes + - restricted-volumes + match: + any: + - resources: + kinds: + - Pod + # because kube-vip prefix is common also for kube-vip-cloud-provider, we need to distinguish bethween the two + selector: + matchLabels: + {{- include "kube-vip.selectorLabels" . | nindent 10 }} + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip-* + - resources: + kinds: + - DaemonSet + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip +{{- end }} \ No newline at end of file diff --git a/config/vsphere-csi-driver/kustomization.yaml b/config/vsphere-csi-driver/kustomization.yaml index 13db989..4f242cd 100644 --- a/config/vsphere-csi-driver/kustomization.yaml +++ b/config/vsphere-csi-driver/kustomization.yaml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization namespace: "{{ .Release.Namespace }}" patchesStrategicMerge: diff --git a/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml b/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml new file mode 100644 index 0000000..1e7f41b --- /dev/null +++ b/config/vsphere-csi-driver/overwrites/templates/pss-exceptions.yaml @@ -0,0 +1,65 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: vsphere-csi-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-privileged-containers + ruleNames: + - autogen-privileged-containers + - privileged-containers + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: disallow-host-ports + ruleNames: + - host-ports-none + - autogen-host-ports-none + - policyName: restrict-volume-types + ruleNames: + - restricted-volumes + - autogen-restricted-volumes + - policyName: require-run-as-nonroot + ruleNames: + - run-as-non-root + - autogen-run-as-non-root + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + - policyName: disallow-capabilities + ruleNames: + - adding-capabilities + - autogen-adding-capabilities + - policyName: disallow-capabilities-strict + ruleNames: + - adding-capabilities-strict + - autogen-adding-capabilities-strict + - require-drop-all + - autogen-require-drop-all + - policyName: disallow-privilege-escalation + ruleNames: + - privilege-escalation + - autogen-privilege-escalation + match: + any: + - resources: + kinds: + - DaemonSet + - Deployment + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - vsphere-csi* +{{- end }} \ No newline at end of file diff --git a/hack/update-csi-chart.sh b/hack/update-csi-chart.sh index 0d3a824..72a306d 100755 --- a/hack/update-csi-chart.sh +++ b/hack/update-csi-chart.sh @@ -4,6 +4,7 @@ set -euo pipefail base_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) chart_dir="./helm/cloud-provider-vsphere/charts/vsphere-csi-driver" +KUSTOMIZE="${1:-kustomize}" cd "$base_dir" @@ -24,7 +25,7 @@ cp -R \ # Customizations -kubectl kustomize "./config/vsphere-csi-driver" -o "./config/vsphere-csi-driver/tmp" +${KUSTOMIZE} build "./config/vsphere-csi-driver" -o "./config/vsphere-csi-driver/tmp" find \ "./config/vsphere-csi-driver/tmp/" \ diff --git a/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml new file mode 100644 index 0000000..197a3fe --- /dev/null +++ b/helm/cloud-provider-vsphere/charts/cloud-provider-for-vsphere/templates/pss-exceptions.yaml @@ -0,0 +1,34 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: cloud-provider-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + match: + any: + - resources: + kinds: + - DaemonSet + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - cloud-provider-for-vsphere* +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml new file mode 100644 index 0000000..7e105f1 --- /dev/null +++ b/helm/cloud-provider-vsphere/charts/kube-vip-cloud-provider/templates/pss-exceptions.yaml @@ -0,0 +1,34 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: kube-vip-cloud-provider-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + match: + any: + - resources: + kinds: + - Deployment + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip-cloud-provider* +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml new file mode 100644 index 0000000..e766875 --- /dev/null +++ b/helm/cloud-provider-vsphere/charts/kube-vip/templates/pss-exceptions.yaml @@ -0,0 +1,65 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: kube-vip-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-capabilities + ruleNames: + - autogen-adding-capabilities + - adding-capabilities + - policyName: disallow-capabilities-strict + ruleNames: + - autogen-require-drop-all + - require-drop-all + - autogen-adding-capabilities-strict + - adding-capabilities-strict + - policyName: disallow-host-namespaces + ruleNames: + - autogen-host-namespaces + - host-namespaces + - policyName: disallow-privilege-escalation + ruleNames: + - autogen-privilege-escalation + - privilege-escalation + - policyName: require-run-as-nonroot + ruleNames: + - autogen-run-as-non-root + - run-as-non-root + - policyName: restrict-seccomp-strict + ruleNames: + - autogen-check-seccomp-strict + - check-seccomp-strict + - policyName: restrict-image-registries + ruleNames: + - autogen-validate-registries + - validate-registries + - policyName: restrict-volume-types + ruleNames: + - autogen-restricted-volumes + - restricted-volumes + match: + any: + - resources: + kinds: + - Pod + # because kube-vip prefix is common also for kube-vip-cloud-provider, we need to distinguish bethween the two + selector: + matchLabels: + {{- include "kube-vip.selectorLabels" . | nindent 10 }} + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip-* + - resources: + kinds: + - DaemonSet + namespaces: + - {{ .Release.Namespace }} + names: + - kube-vip +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml new file mode 100644 index 0000000..1e7f41b --- /dev/null +++ b/helm/cloud-provider-vsphere/charts/vsphere-csi-driver/templates/pss-exceptions.yaml @@ -0,0 +1,65 @@ +{{- if .Values.global.kyvernoPolicyExceptions.enabled }} +apiVersion: kyverno.io/v2alpha1 +kind: PolicyException +metadata: + name: vsphere-csi-exceptions + namespace: {{ .Release.Namespace }} + labels: + {{- include "labels.common" . | nindent 4 }} +spec: + exceptions: + - policyName: disallow-privileged-containers + ruleNames: + - autogen-privileged-containers + - privileged-containers + - policyName: disallow-host-path + ruleNames: + - autogen-host-path + - host-path + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + - policyName: disallow-host-ports + ruleNames: + - host-ports-none + - autogen-host-ports-none + - policyName: restrict-volume-types + ruleNames: + - restricted-volumes + - autogen-restricted-volumes + - policyName: require-run-as-nonroot + ruleNames: + - run-as-non-root + - autogen-run-as-non-root + - policyName: restrict-seccomp-strict + ruleNames: + - check-seccomp-strict + - autogen-check-seccomp-strict + - policyName: disallow-capabilities + ruleNames: + - adding-capabilities + - autogen-adding-capabilities + - policyName: disallow-capabilities-strict + ruleNames: + - adding-capabilities-strict + - autogen-adding-capabilities-strict + - require-drop-all + - autogen-require-drop-all + - policyName: disallow-privilege-escalation + ruleNames: + - privilege-escalation + - autogen-privilege-escalation + match: + any: + - resources: + kinds: + - DaemonSet + - Deployment + - ReplicaSet + - Pod + namespaces: + - {{ .Release.Namespace }} + names: + - vsphere-csi* +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/templates/_helpers.tpl b/helm/cloud-provider-vsphere/templates/_helpers.tpl index 5ef4ee3..508ff79 100644 --- a/helm/cloud-provider-vsphere/templates/_helpers.tpl +++ b/helm/cloud-provider-vsphere/templates/_helpers.tpl @@ -23,3 +23,29 @@ app.kubernetes.io/version: {{ .Chart.Version | quote }} application.giantswarm.io/team: {{ index .Chart.Annotations "application.giantswarm.io/team" | quote }} helm.sh/chart: {{ include "chart" . | quote }} {{- end -}} + +{{/* +CRD installation labels +*/}} +{{- define "crdInstall" -}} +{{- printf "%s-%s" ( include "name" . ) "crd-install" | replace "+" "_" | trimSuffix "-" -}} +{{- end -}} + +{{- define "crdInstallAnnotations" -}} +"helm.sh/hook": "pre-install,pre-upgrade" +"helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed" +{{- end -}} + +{{/* Create a label which can be used to select any orphaned crd-install hook resources */}} +{{- define "crdInstallSelector" -}} +{{- printf "%s" "crd-install-hook" -}} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "labels.crdSelector" -}} +app.kubernetes.io/component: {{ include "crdInstall" . | quote }} +app.kubernetes.io/name: {{ include "name" . | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- end -}} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-ciliumnetworkpolicy.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-ciliumnetworkpolicy.yaml new file mode 100644 index 0000000..3146e32 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-ciliumnetworkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.global.crdInstall.enabled }} +{{- if .Values.global.ciliumNetworkPolicy.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-12" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +spec: + endpointSelector: + matchLabels: + {{- include "labels.crdSelector" . | nindent 6 }} + egress: + - toEntities: + - kube-apiserver +{{- end }} +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml new file mode 100644 index 0000000..f2c5f86 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-configmap.yaml @@ -0,0 +1,27 @@ +{{/* +We have to create individual configmaps for each CRD - they exceed the total +allowed length for a configmap if they are combined. +*/}} +{{ $currentScope := . }} +{{- if .Values.global.crdInstall.enabled }} + {{- range $path, $_ := .Files.Glob "files/**" }} + {{- with $currentScope }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-15" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +data: + content: | +{{ tpl ($.Files.Get $path) . | indent 4 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml new file mode 100644 index 0000000..61d6af5 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-job.yaml @@ -0,0 +1,75 @@ +{{- if .Values.global.crdInstall.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-11" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +spec: + template: + metadata: + labels: + {{- include "labels.crdSelector" . | nindent 8 }} + spec: + serviceAccountName: {{ include "crdInstall" . }} + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + {{- with .Values.podSecurityContext }} + {{- . | toYaml | nindent 8 }} + {{- end }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + containers: + - name: kubectl + image: "{{ .Values.crdInstall.kubectl.registry }}/{{ .Values.crdInstall.kubectl.image }}:{{ .Values.crdInstall.kubectl.tag }}" + command: + - sh + - -c + - | + set -o errexit ; set -o xtrace ; set -o nounset + # piping stderr to stdout means kubectl's errors are surfaced + # in the pod's logs. + kubectl apply -f /data/ 2>&1 + securityContext: + readOnlyRootFilesystem: true + {{- with .Values.containerSecurityContext }} + {{- . | toYaml | nindent 10 }} + {{- end }} + volumeMounts: +{{- range $path, $_ := .Files.Glob "files/**" }} + - name: {{ $path | base | trimSuffix ".yaml" }} + mountPath: /data/{{ $path | base }} + subPath: {{ $path | base }} +{{- end }} + resources: {{- toYaml .Values.crdInstall.resources | nindent 10 }} + volumes: +{{ $currentScope := . }} +{{- range $path, $_ := .Files.Glob "files/**" }} + {{- with $currentScope }} + - name: {{ $path | base | trimSuffix ".yaml" }} + configMap: + name: {{ include "crdInstall" . }}-{{ $path | base | trimSuffix ".yaml" }} + items: + - key: content + path: {{ $path | base }} +{{- end }} +{{- end }} + restartPolicy: Never + backoffLimit: 10 +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml new file mode 100644 index 0000000..97e899a --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-netpol.yaml @@ -0,0 +1,36 @@ +{{- if .Values.global.crdInstall.enabled }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-17" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +spec: + podSelector: + matchLabels: + {{- include "labels.crdSelector" . | nindent 6 }} + # allow egress traffic to the Kubernetes API + egress: + - ports: + - port: 443 + protocol: TCP + # legacy port kept for compatibility + - port: 6443 + protocol: TCP + to: + {{- range tuple "10.0.0.0/8" "172.16.0.0/12" "192.168.0.0/16" "100.64.0.0/10" }} + - ipBlock: + cidr: {{ . }} + {{- end }} + # deny ingress traffic + ingress: [] + policyTypes: + - Egress + - Ingress +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-psp.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-psp.yaml new file mode 100644 index 0000000..522f8c4 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-psp.yaml @@ -0,0 +1,40 @@ +{{- if .Values.global.crdInstall.enabled }} +{{- if not .Values.global.podSecurityStandards.enforced }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "crdInstall" . }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-16" + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +spec: + privileged: false + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + volumes: + - 'configMap' + hostPID: false + hostIPC: false + hostNetwork: false + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml new file mode 100644 index 0000000..5d51797 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-rbac.yaml @@ -0,0 +1,64 @@ +{{- if .Values.global.crdInstall.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-13" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +rules: +- apiGroups: + - "" + resources: + - jobs + verbs: + - create + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - patch +{{- if not .Values.global.podSecurityStandards.enforced }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - {{ include "crdInstall" . }} + verbs: + - use +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-12" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + {{- include "labels.crdSelector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "crdInstall" . }} +subjects: + - kind: ServiceAccount + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} \ No newline at end of file diff --git a/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml b/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml new file mode 100644 index 0000000..57edf33 --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/crd-install/crd-serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.global.crdInstall.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "crdInstall" . }} + namespace: {{ .Release.Namespace }} + annotations: + # create hook dependencies in the right order + "helm.sh/hook-weight": "-14" + {{- include "crdInstallAnnotations" . | nindent 4 }} + labels: + app.kubernetes.io/component: {{ include "crdInstall" . | quote }} + {{- include "labels.selector" . | nindent 4 }} + role: {{ include "crdInstallSelector" . | quote }} +{{- end }} diff --git a/helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml b/helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml new file mode 100644 index 0000000..42f111c --- /dev/null +++ b/helm/cloud-provider-vsphere/templates/files/policyexceptions.yaml @@ -0,0 +1,523 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + labels: + app.kubernetes.io/name: kyverno + app.kubernetes.io/part-of: kyverno + name: policyexceptions.kyverno.io +spec: + conversion: + strategy: None + group: kyverno.io + names: + categories: + - kyverno + kind: PolicyException + listKind: PolicyExceptionList + plural: policyexceptions + shortNames: + - polex + singular: policyexception + scope: Namespaced + versions: + - name: v2alpha1 + schema: + openAPIV3Schema: + description: PolicyException declares resources to be excluded from specified + policies. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec declares policy exception behaviors. + properties: + background: + description: Background controls if exceptions are applied to existing + policies during a background scan. Optional. Default value is "true". + The value must be set to "false" if the policy rule uses variables + that are only available in the admission review request (e.g. user + name). + type: boolean + exceptions: + description: Exceptions is a list policy/rules to be excluded + items: + description: Exception stores infos about a policy and rules + properties: + policyName: + description: PolicyName identifies the policy to which the exception + is applied. The policy name uses the format / + unless it references a ClusterPolicy. + type: string + ruleNames: + description: RuleNames identifies the rules to which the exception + is applied. + items: + type: string + type: array + required: + - policyName + - ruleNames + type: object + type: array + match: + description: Match defines match clause used to check if a resource + applies to the exception + properties: + all: + description: All allows specifying resources which will be ANDed + items: + description: ResourceFilter allow users to "AND" or "OR" between + resources + properties: + clusterRoles: + description: ClusterRoles is the list of cluster-wide role + names for the user. + items: + type: string + type: array + resources: + description: ResourceDescription contains information about + the resource being created or modified. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a map of annotations (key-value + pairs of type string). Annotation keys and values + support the wildcard characters "*" (matches zero + or many characters) and "?" (matches at least one + character). + type: object + kinds: + description: Kinds is a list of resource kinds. + items: + type: string + type: array + name: + description: 'Name is the name of the resource. The + name supports wildcard characters "*" (matches zero + or many characters) and "?" (at least one character). + NOTE: "Name" is being deprecated in favor of "Names".' + type: string + names: + description: Names are the names of the resources. Each + name supports wildcard characters "*" (matches zero + or many characters) and "?" (at least one character). + items: + type: string + type: array + namespaceSelector: + description: 'NamespaceSelector is a label selector + for the resource namespace. Label keys and values + in `matchLabels` support the wildcard characters `*` + (matches zero or many characters) and `?` (matches + one character).Wildcards allows writing label selectors + like ["storage.k8s.io/*": "*"]. Note that using ["*" + : "*"] matches any key and value but does not match + an empty label set.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: Namespaces is a list of namespaces names. + Each name supports wildcard characters "*" (matches + zero or many characters) and "?" (at least one character). + items: + type: string + type: array + operations: + description: Operations can contain values ["CREATE, + "UPDATE", "CONNECT", "DELETE"], which are used to + match a specific action. + items: + description: AdmissionOperation can have one of the + values CREATE, UPDATE, CONNECT, DELETE, which are + used to match a specific action. + enum: + - CREATE + - CONNECT + - UPDATE + - DELETE + type: string + type: array + selector: + description: 'Selector is a label selector. Label keys + and values in `matchLabels` support the wildcard characters + `*` (matches zero or many characters) and `?` (matches + one character). Wildcards allows writing label selectors + like ["storage.k8s.io/*": "*"]. Note that using ["*" + : "*"] matches any key and value but does not match + an empty label set.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + roles: + description: Roles is the list of namespaced role names + for the user. + items: + type: string + type: array + subjects: + description: Subjects is the list of subject names like + users, user groups, and service accounts. + items: + description: Subject contains a reference to the object + or user identities a role binding applies to. This + can either hold a direct API object reference, or a + value for non-objects such as user and group names. + properties: + apiGroup: + description: APIGroup holds the API group of the referenced + subject. Defaults to "" for ServiceAccount subjects. + Defaults to "rbac.authorization.k8s.io" for User + and Group subjects. + type: string + kind: + description: Kind of object being referenced. Values + defined by this API group are "User", "Group", and + "ServiceAccount". If the Authorizer does not recognized + the kind value, the Authorizer should report an + error. + type: string + name: + description: Name of the object being referenced. + type: string + namespace: + description: Namespace of the referenced object. If + the object kind is non-namespace, such as "User" + or "Group", and this value is not empty the Authorizer + should report an error. + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + type: array + type: object + type: array + any: + description: Any allows specifying resources which will be ORed + items: + description: ResourceFilter allow users to "AND" or "OR" between + resources + properties: + clusterRoles: + description: ClusterRoles is the list of cluster-wide role + names for the user. + items: + type: string + type: array + resources: + description: ResourceDescription contains information about + the resource being created or modified. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a map of annotations (key-value + pairs of type string). Annotation keys and values + support the wildcard characters "*" (matches zero + or many characters) and "?" (matches at least one + character). + type: object + kinds: + description: Kinds is a list of resource kinds. + items: + type: string + type: array + name: + description: 'Name is the name of the resource. The + name supports wildcard characters "*" (matches zero + or many characters) and "?" (at least one character). + NOTE: "Name" is being deprecated in favor of "Names".' + type: string + names: + description: Names are the names of the resources. Each + name supports wildcard characters "*" (matches zero + or many characters) and "?" (at least one character). + items: + type: string + type: array + namespaceSelector: + description: 'NamespaceSelector is a label selector + for the resource namespace. Label keys and values + in `matchLabels` support the wildcard characters `*` + (matches zero or many characters) and `?` (matches + one character).Wildcards allows writing label selectors + like ["storage.k8s.io/*": "*"]. Note that using ["*" + : "*"] matches any key and value but does not match + an empty label set.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: Namespaces is a list of namespaces names. + Each name supports wildcard characters "*" (matches + zero or many characters) and "?" (at least one character). + items: + type: string + type: array + operations: + description: Operations can contain values ["CREATE, + "UPDATE", "CONNECT", "DELETE"], which are used to + match a specific action. + items: + description: AdmissionOperation can have one of the + values CREATE, UPDATE, CONNECT, DELETE, which are + used to match a specific action. + enum: + - CREATE + - CONNECT + - UPDATE + - DELETE + type: string + type: array + selector: + description: 'Selector is a label selector. Label keys + and values in `matchLabels` support the wildcard characters + `*` (matches zero or many characters) and `?` (matches + one character). Wildcards allows writing label selectors + like ["storage.k8s.io/*": "*"]. Note that using ["*" + : "*"] matches any key and value but does not match + an empty label set.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + roles: + description: Roles is the list of namespaced role names + for the user. + items: + type: string + type: array + subjects: + description: Subjects is the list of subject names like + users, user groups, and service accounts. + items: + description: Subject contains a reference to the object + or user identities a role binding applies to. This + can either hold a direct API object reference, or a + value for non-objects such as user and group names. + properties: + apiGroup: + description: APIGroup holds the API group of the referenced + subject. Defaults to "" for ServiceAccount subjects. + Defaults to "rbac.authorization.k8s.io" for User + and Group subjects. + type: string + kind: + description: Kind of object being referenced. Values + defined by this API group are "User", "Group", and + "ServiceAccount". If the Authorizer does not recognized + the kind value, the Authorizer should report an + error. + type: string + name: + description: Name of the object being referenced. + type: string + namespace: + description: Namespace of the referenced object. If + the object kind is non-namespace, such as "User" + or "Group", and this value is not empty the Authorizer + should report an error. + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + type: array + type: object + type: array + type: object + required: + - exceptions + - match + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/helm/cloud-provider-vsphere/values.schema.json b/helm/cloud-provider-vsphere/values.schema.json index 33b33b4..1a22f81 100644 --- a/helm/cloud-provider-vsphere/values.schema.json +++ b/helm/cloud-provider-vsphere/values.schema.json @@ -15,6 +15,28 @@ } } }, + "crdInstall": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "kubectl": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "registry": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + } + } + }, "global": { "type": "object", "properties": { @@ -49,6 +71,22 @@ "type": "string" } } + }, + "kyvernoPolicyExceptions": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "podSecurityStandards": { + "type": "object", + "properties": { + "enforced": { + "type": "boolean" + } + } } } }, diff --git a/helm/cloud-provider-vsphere/values.yaml b/helm/cloud-provider-vsphere/values.yaml index 3c964c2..a22b576 100644 --- a/helm/cloud-provider-vsphere/values.yaml +++ b/helm/cloud-provider-vsphere/values.yaml @@ -1,4 +1,8 @@ global: + + ciliumNetworkPolicy: + enabled: true + config: enabled: true clusterId: "" @@ -11,6 +15,19 @@ global: # openssl s_client -connect :443 < /dev/null 2>/dev/null | openssl x509 -fingerprint -sha1 -noout thumbprint: "" + crdInstall: + enabled: true + kubectl: + image: "giantswarm/kubectl" + registry: quay.io + tag: 1.25.10 + + kyvernoPolicyExceptions: + enabled: true + + podSecurityStandards: + enforced: false + cloud-provider-for-vsphere: daemonset: image: docker.io/giantswarm/cpi-vsphere-manager @@ -37,3 +54,17 @@ kube-vip-cloud-provider: image: repository: docker.io/giantswarm/kube-vip-cloud-provider tag: "v0.0.4" + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault