From e1880704904d62e3f9f303f9c96edeac57aaecbd Mon Sep 17 00:00:00 2001 From: deepakm-ntnx Date: Tue, 19 Dec 2023 14:52:03 -0800 Subject: [PATCH] added first jsonPatches in clusterclass template and cluster topology --- Makefile | 37 +- templates/base-root/ccm-patch.yaml | 11 - templates/base-root/kustomization.yaml | 4 - templates/base-root/nutanix-ccm-crs.yaml | 2 +- templates/base/ccm-patch.yaml | 14 +- templates/{base-root => base}/kct.yaml | 0 templates/base/kustomization.yaml | 1 + templates/cluster-template-clusterclass.yaml | 58 +- templates/cluster-template-csi.yaml | 2 +- templates/cluster-template-topology.yaml | 5 +- templates/cluster-template.yaml | 2 +- templates/clusterclass/clusterclass.yaml | 46 +- templates/clusterclass/kcpt.yaml | 6 - templates/clusterclass/kct.yaml | 19 + templates/clusterclass/kustomization.yaml | 3 +- templates/topology/cluster-with-topology.yaml | 7 +- .../infrastructure-nutanix/ccm-update.yaml | 6 +- .../v1.2.4/base/cluster-with-kcp.yaml | 2 +- .../v1.2.4/base/cm.yaml | 2 +- .../v1alpha4/bases/cm.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 3 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../v1beta1/clusterclass-e2e.yaml | 624 ++++++++++++++++++ 29 files changed, 796 insertions(+), 74 deletions(-) delete mode 100644 templates/base-root/ccm-patch.yaml rename templates/{base-root => base}/kct.yaml (100%) create mode 100644 templates/clusterclass/kct.yaml create mode 100644 test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml diff --git a/Makefile b/Makefile index bcea15caa5..877e0aa80b 100644 --- a/Makefile +++ b/Makefile @@ -177,6 +177,8 @@ FLAVOR ?= e2e TEST_NAMESPACE=capx-test-ns TEST_CLUSTER_NAME=mycluster +TEST_CLUSTER_CLASS_NAME=my-clusterclass +TEST_TOPOLOGY_CLUSTER_NAME=my-cc-cluster # set ginkgo focus flags, if any ifneq ($(strip $(GINKGO_FOCUS)),) @@ -323,6 +325,7 @@ cluster-e2e-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1 $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml + $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates without kubeproxy # v1alpha4 @@ -342,6 +345,7 @@ cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates wi $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml + $(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml cluster-templates: $(KUSTOMIZE) ## Generate cluster templates for all flavors $(KUSTOMIZE) build $(TEMPLATES_DIR)/base > $(TEMPLATES_DIR)/cluster-template.yaml @@ -415,32 +419,37 @@ list-workload-resources: ## Run kubectl queries to get all capx workload related .PHONY: test-cc-cluster-create test-cc-cluster-create: cluster-templates - clusterctl generate cluster cc-test --from ./templates/cluster-template-clusterclass.yaml -n $(TEST_NAMESPACE) > cc-test.yaml - clusterctl generate cluster cluster-topology --from ./templates/cluster-template-topology.yaml -n $(TEST_NAMESPACE) > cluster-topology.yaml + clusterctl generate cluster ${TEST_CLUSTER_CLASS_NAME} --from ./templates/cluster-template-clusterclass.yaml -n $(TEST_NAMESPACE) > ${TEST_CLUSTER_CLASS_NAME}.yaml + clusterctl generate cluster ${TEST_TOPOLOGY_CLUSTER_NAME} --from ./templates/cluster-template-topology.yaml -n $(TEST_NAMESPACE) > ${TEST_TOPOLOGY_CLUSTER_NAME}.yaml kubectl create ns $(TEST_NAMESPACE) --dry-run=client -oyaml | kubectl apply --server-side -f - - kubectl apply --server-side -f ./cc-test.yaml - kubectl apply --server-side -f ./cluster-topology.yaml + kubectl apply --server-side -f ./${TEST_CLUSTER_CLASS_NAME}.yaml + kubectl apply --server-side -f ./${TEST_TOPOLOGY_CLUSTER_NAME}.yaml .PHONY: test-cc-cluster-delete test-cc-cluster-delete: - kubectl -n $(TEST_NAMESPACE) delete cluster cluster-topology --ignore-not-found - kubectl -n $(TEST_NAMESPACE) delete secret cluster-topology --ignore-not-found - kubectl -n $(TEST_NAMESPACE) delete cm user-ca-bundle --ignore-not-found - rm cluster-topology.yaml || true - rm cc-test.yaml || true + kubectl -n $(TEST_NAMESPACE) delete cluster ${TEST_TOPOLOGY_CLUSTER_NAME} --ignore-not-found + kubectl -n $(TEST_NAMESPACE) delete secret ${TEST_TOPOLOGY_CLUSTER_NAME} --ignore-not-found + kubectl -n $(TEST_NAMESPACE) delete cm ${TEST_TOPOLOGY_CLUSTER_NAME}-pc-trusted-ca-bundle --ignore-not-found + rm ${TEST_TOPOLOGY_CLUSTER_NAME}.yaml || true + rm ${TEST_CLUSTER_CLASS_NAME}.yaml || true +.PHONY: generate-cc-cluster-kubeconfig +generate-cc-cluster-kubeconfig: + kubectl -n ${TEST_NAMESPACE} get secret ${TEST_TOPOLOGY_CLUSTER_NAME}-kubeconfig -o json | jq -r .data.value | base64 --decode > ${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig + +.PHONY: test-cc-cluster-install-cni +test-cc-cluster-install-cni: generate-cc-cluster-kubeconfig + kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig apply -f https://raw.githubusercontent.com/nutanix-cloud-native/cluster-api-provider-nutanix/main/test/e2e/data/cni/calico/calico.yaml .PHONY: list-cc-cluster-resources -list-cc-cluster-resources: +list-cc-cluster-resources: generate-cc-cluster-kubeconfig kubectl -n capx-system get endpoints kubectl get crd | grep nutanix kubectl get cluster-api -A kubectl -n $(TEST_NAMESPACE) get Cluster,NutanixCluster,Machine,NutanixMachine,KubeAdmControlPlane,MachineHealthCheck,nodes kubectl get ValidatingWebhookConfiguration,MutatingWebhookConfiguration -A - kubectl -n ${TEST_NAMESPACE} get secret cluster-topology-kubeconfig -o json | jq -r .data.value | base64 --decode > cluster-topology.workload.kubeconfig - kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get nodes,ns - kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get nodes,ns - kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get pods -A + kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig get nodes,ns + kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig get pods -A .PHONY: ginkgo-help ginkgo-help: diff --git a/templates/base-root/ccm-patch.yaml b/templates/base-root/ccm-patch.yaml deleted file mode 100644 index 7ac9eafc72..0000000000 --- a/templates/base-root/ccm-patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-kcfg-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external diff --git a/templates/base-root/kustomization.yaml b/templates/base-root/kustomization.yaml index 7cef6df9e6..9ee5d03569 100644 --- a/templates/base-root/kustomization.yaml +++ b/templates/base-root/kustomization.yaml @@ -12,9 +12,5 @@ configMapGenerator: bases: - ./secret.yaml - ./cm.yaml - - ./kct.yaml - ./nutanix-ccm-crs.yaml - ./nutanix-ccm-secret.yaml - -patchesStrategicMerge: -- ./ccm-patch.yaml diff --git a/templates/base-root/nutanix-ccm-crs.yaml b/templates/base-root/nutanix-ccm-crs.yaml index 188d39347c..ede23956fa 100644 --- a/templates/base-root/nutanix-ccm-crs.yaml +++ b/templates/base-root/nutanix-ccm-crs.yaml @@ -12,7 +12,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: v1 diff --git a/templates/base/ccm-patch.yaml b/templates/base/ccm-patch.yaml index 0b0c0a3bbe..4a7de496e9 100644 --- a/templates/base/ccm-patch.yaml +++ b/templates/base/ccm-patch.yaml @@ -25,4 +25,16 @@ kind: Cluster metadata: labels: ccm: "nutanix" - name: "${CLUSTER_NAME}" \ No newline at end of file + name: "${CLUSTER_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-kcfg-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external \ No newline at end of file diff --git a/templates/base-root/kct.yaml b/templates/base/kct.yaml similarity index 100% rename from templates/base-root/kct.yaml rename to templates/base/kct.yaml diff --git a/templates/base/kustomization.yaml b/templates/base/kustomization.yaml index 0aa4b88040..384fb80031 100644 --- a/templates/base/kustomization.yaml +++ b/templates/base/kustomization.yaml @@ -8,6 +8,7 @@ bases: - ./nutanix-cluster.yaml - ./cluster-without-topology.yaml - ./kcp.yaml + - ./kct.yaml - ./nmt.yaml - ./md.yaml - ./mhc.yaml diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml index f087681e93..8350677e40 100644 --- a/templates/cluster-template-clusterclass.yaml +++ b/templates/cluster-template-clusterclass.yaml @@ -254,7 +254,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -276,12 +276,6 @@ spec: preKubeadmCommands: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - users: - - lockPassword: false - name: capiuser - sshAuthorizedKeys: - - ${NUTANIX_SSH_AUTHORIZED_KEY} - sudo: ALL=(ALL) NOPASSWD:ALL verbosity: 10 --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -330,9 +324,51 @@ spec: kind: NutanixClusterTemplate name: ${CLUSTER_CLASS_NAME}-nct namespace: ${NAMESPACE} + patches: + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: add-ssh-user + variables: + - name: sshKey + required: false + schema: + openAPIV3Schema: + description: Public key to SSH onto the cluster nodes. + type: string workers: machineDeployments: - - class: default-worker + - class: ${CLUSTER_CLASS_NAME}-worker machineHealthCheck: maxUnhealthy: 40% nodeStartupTimeout: 10m @@ -480,12 +516,6 @@ spec: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" useExperimentalRetryJoin: true - users: - - lockPassword: false - name: capiuser - sshAuthorizedKeys: - - ${NUTANIX_SSH_AUTHORIZED_KEY} - sudo: ALL=(ALL) NOPASSWD:ALL verbosity: 10 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-csi.yaml b/templates/cluster-template-csi.yaml index 9127c12ec3..7036d7f319 100644 --- a/templates/cluster-template-csi.yaml +++ b/templates/cluster-template-csi.yaml @@ -1573,7 +1573,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: addons.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index 1ec52e55d0..ab0ac4a03f 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -20,10 +20,13 @@ spec: controlPlane: metadata: {} replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: sshKey + value: ${NUTANIX_SSH_AUTHORIZED_KEY} version: ${KUBERNETES_VERSION} workers: machineDeployments: - - class: default-worker + - class: ${CLUSTER_CLASS_NAME}-worker metadata: {} name: md-0 replicas: ${WORKER_MACHINE_COUNT} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 81ee29ce6f..0b7dc11a04 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -254,7 +254,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/clusterclass/clusterclass.yaml b/templates/clusterclass/clusterclass.yaml index 498eb7ca53..c256b50f18 100644 --- a/templates/clusterclass/clusterclass.yaml +++ b/templates/clusterclass/clusterclass.yaml @@ -39,7 +39,7 @@ spec: timeout: 300s workers: machineDeployments: - - class: default-worker + - class: ${CLUSTER_CLASS_NAME}-worker template: bootstrap: ref: @@ -80,4 +80,46 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixClusterTemplate name: ${CLUSTER_CLASS_NAME}-nct - namespace: ${NAMESPACE} \ No newline at end of file + namespace: ${NAMESPACE} + patches: + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: add-ssh-user + variables: + - name: sshKey + required: false + schema: + openAPIV3Schema: + description: Public key to SSH onto the cluster nodes. + type: string \ No newline at end of file diff --git a/templates/clusterclass/kcpt.yaml b/templates/clusterclass/kcpt.yaml index 9c0b4dec68..76f5d36e7f 100644 --- a/templates/clusterclass/kcpt.yaml +++ b/templates/clusterclass/kcpt.yaml @@ -108,10 +108,4 @@ spec: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" useExperimentalRetryJoin: true - users: - - name: capiuser - lockPassword: false - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - ${NUTANIX_SSH_AUTHORIZED_KEY} verbosity: 10 \ No newline at end of file diff --git a/templates/clusterclass/kct.yaml b/templates/clusterclass/kct.yaml new file mode 100644 index 0000000000..74fc727819 --- /dev/null +++ b/templates/clusterclass/kct.yaml @@ -0,0 +1,19 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-kcfg-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + postKubeadmCommands: + - echo "after kubeadm call" > /var/log/postkubeadm.log + preKubeadmCommands: + - echo "before kubeadm call" > /var/log/prekubeadm.log + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + verbosity: 10 diff --git a/templates/clusterclass/kustomization.yaml b/templates/clusterclass/kustomization.yaml index 885e0cadbc..9f88a8f1b4 100644 --- a/templates/clusterclass/kustomization.yaml +++ b/templates/clusterclass/kustomization.yaml @@ -9,4 +9,5 @@ bases: - ./clusterclass.yaml - ./nmt-cp.yaml - ./nmt-md.yaml - - ./kcpt.yaml \ No newline at end of file + - ./kcpt.yaml + - ./kct.yaml \ No newline at end of file diff --git a/templates/topology/cluster-with-topology.yaml b/templates/topology/cluster-with-topology.yaml index 83668fdffa..2d8fbc0a32 100644 --- a/templates/topology/cluster-with-topology.yaml +++ b/templates/topology/cluster-with-topology.yaml @@ -19,7 +19,10 @@ spec: metadata: {} workers: machineDeployments: - - class: default-worker + - class: ${CLUSTER_CLASS_NAME}-worker name: md-0 replicas: ${WORKER_MACHINE_COUNT} - metadata: {} \ No newline at end of file + metadata: {} + variables: + - name: sshKey + value: '${NUTANIX_SSH_AUTHORIZED_KEY}' \ No newline at end of file diff --git a/test/e2e/data/infrastructure-nutanix/ccm-update.yaml b/test/e2e/data/infrastructure-nutanix/ccm-update.yaml index 598e5557f4..aad98acd5a 100644 --- a/test/e2e/data/infrastructure-nutanix/ccm-update.yaml +++ b/test/e2e/data/infrastructure-nutanix/ccm-update.yaml @@ -5,7 +5,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle namespace: kube-system binaryData: ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} @@ -37,7 +37,7 @@ data: }, "additionalTrustBundle": { "kind": "ConfigMap", - "name": "user-ca-bundle", + "name": "${CLUSTER_NAME}-pc-trusted-ca-bundle", "namespace": "kube-system" } }, @@ -245,5 +245,5 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle strategy: ApplyOnce diff --git a/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cluster-with-kcp.yaml index b87787daee..4b9fe7f647 100644 --- a/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cluster-with-kcp.yaml @@ -12,7 +12,7 @@ spec: name: "${CLUSTER_NAME}" kind: Secret additionalTrustBundle: - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle kind: ConfigMap controlPlaneEndpoint: host: "${CONTROL_PLANE_ENDPOINT_IP_V124}" diff --git a/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cm.yaml b/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cm.yaml index ff4d33af76..5bf1d2f7ef 100644 --- a/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cm.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1.2.4/base/cm.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle namespace: "${NAMESPACE}" binaryData: ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} diff --git a/test/e2e/data/infrastructure-nutanix/v1alpha4/bases/cm.yaml b/test/e2e/data/infrastructure-nutanix/v1alpha4/bases/cm.yaml index a968f21831..a7332f1690 100644 --- a/test/e2e/data/infrastructure-nutanix/v1alpha4/bases/cm.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1alpha4/bases/cm.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: user-ca-bundle + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle namespace: "${NAMESPACE}" data: ca.crt: "${NUTANIX_ADDITIONAL_TRUST_BUNDLE=''}" diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml index c30e37f04a..90dd08d63f 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/md.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml index 71a27509e5..c5ca9c48fd 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/nmt.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml index dea9eb2fe0..e860f5a55e 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/nmt.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml index c25d222ad7..0af1938185 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/md.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml index e53588e823..398e1b0d1e 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/md.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml index 3b5f896c10..d920b084ca 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml @@ -1,7 +1,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base/md.yaml @@ -19,5 +19,4 @@ configMapGenerator: patchesStrategicMerge: - ../base/cni-patch.yaml - - ../../../../../../templates/base-root/ccm-patch.yaml - ../../../../../../templates/base/ccm-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml index 204624c351..16c9404d47 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/nmt.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml index 1fd4e9ed8f..cb02c7b719 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml @@ -2,7 +2,7 @@ bases: - ../../../../../../templates/base/nutanix-cluster.yaml - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - - ../../../../../../templates/base-root/kct.yaml + - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base-root/secret.yaml - ../../../../../../templates/base-root/cm.yaml - ../../../../../../templates/base/md.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml new file mode 100644 index 0000000000..dd078d65e9 --- /dev/null +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml @@ -0,0 +1,624 @@ +apiVersion: v1 +binaryData: + ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + namespace: ${NAMESPACE} +--- +apiVersion: v1 +data: + nutanix-ccm.yaml: | + # Source: nutanix-cloud-provider/templates/rbac.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + # Source: nutanix-cloud-provider/templates/cm.yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nutanix-config + namespace: kube-system + data: + nutanix_config.json: |- + { + "prismCentral": { + "address": "${NUTANIX_ENDPOINT}", + "port": ${NUTANIX_PORT=9440}, + "insecure": ${NUTANIX_INSECURE=false}, + "credentialRef": { + "kind": "secret", + "name": "nutanix-creds", + "namespace": "kube-system" + }, + "additionalTrustBundle": { + "kind": "ConfigMap", + "name": "${CLUSTER_NAME}-pc-trusted-ca-bundle", + "namespace": "kube-system" + } + }, + "enableCustomLabeling": ${CCM_CUSTOM_LABEL=false}, + "topologyDiscovery": { + "type": "Prism" + } + } + --- + # Source: nutanix-cloud-provider/templates/rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + --- + # Source: nutanix-cloud-provider/templates/rbac.yaml + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + # Source: nutanix-cloud-provider/templates/cloud-provider-nutanix-deployment.yaml + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + name: nutanix-cloud-controller-manager + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + strategy: + type: Recreate + template: + metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/control-plane: "" + serviceAccountName: cloud-controller-manager + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + topologyKey: kubernetes.io/hostname + dnsPolicy: Default + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 120 + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 120 + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + containers: + - image: "${CCM_REPO=ghcr.io/nutanix-cloud-native/cloud-provider-nutanix/controller}:${CCM_TAG=v0.3.1}" + imagePullPolicy: IfNotPresent + name: nutanix-cloud-controller-manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "--leader-elect=true" + - "--cloud-config=/etc/cloud/nutanix_config.json" + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - mountPath: /etc/cloud + name: nutanix-config-volume + readOnly: true + volumes: + - name: nutanix-config-volume + configMap: + name: nutanix-config +kind: ConfigMap +metadata: + name: nutanix-ccm + namespace: ${NAMESPACE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +stringData: + credentials: "[\n {\n \"type\": \"basic_auth\", \n \"data\": { \n \"prismCentral\":{\n + \ \"username\": \"${NUTANIX_USER}\", \n \"password\": \"${NUTANIX_PASSWORD}\"\n + \ }\n }\n }\n]\n" +--- +apiVersion: v1 +kind: Secret +metadata: + name: nutanix-ccm-secret + namespace: ${NAMESPACE} +stringData: + nutanix-ccm-secret.yaml: "apiVersion: v1\nkind: Secret\nmetadata:\n name: nutanix-creds\n + \ namespace: kube-system\nstringData:\n credentials: |\n [\n {\n \"type\": + \"basic_auth\", \n \"data\": { \n \"prismCentral\":{\n \"username\": + \"${NUTANIX_USER}\",\n \"password\": \"${NUTANIX_PASSWORD}\"\n },\n + \ \"prismElements\": null\n }\n }\n ]\n" +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: nutanix-ccm-crs + namespace: ${NAMESPACE} +spec: + clusterSelector: + matchLabels: + ccm: nutanix + resources: + - kind: ConfigMap + name: nutanix-ccm + - kind: Secret + name: nutanix-ccm-secret + - kind: ConfigMap + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-kcfg-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + postKubeadmCommands: + - echo "after kubeadm call" > /var/log/postkubeadm.log + preKubeadmCommands: + - echo "before kubeadm call" > /var/log/prekubeadm.log + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + verbosity: 10 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: nutanix + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: ${CLUSTER_NAME}-crs-cni + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 172.20.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 172.19.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + controlPlane: + metadata: {} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: sshKey + value: ${NUTANIX_SSH_AUTHORIZED_KEY} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: ${CLUSTER_CLASS_NAME}-worker + metadata: {} + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: ${NAMESPACE} +spec: + controlPlane: + machineHealthCheck: + maxUnhealthy: 40% + nodeStartupTimeout: 10m + unhealthyConditions: + - status: "False" + timeout: 300s + type: Ready + - status: Unknown + timeout: 300s + type: Ready + - status: "True" + timeout: 300s + type: MemoryPressure + - status: "True" + timeout: 300s + type: DiskPressure + - status: "True" + timeout: 300s + type: PIDPressure + - status: "True" + timeout: 300s + type: NetworkUnavailable + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: NutanixMachineTemplate + name: ${CLUSTER_CLASS_NAME}-cp-nmt + namespace: ${NAMESPACE} + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: ${CLUSTER_CLASS_NAME}-kcpt + namespace: ${NAMESPACE} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: NutanixClusterTemplate + name: ${CLUSTER_CLASS_NAME}-nct + namespace: ${NAMESPACE} + patches: + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/users + valueFrom: + template: | + - name: capxuser + lockPassword: false + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - '{{ .sshKey }}' + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: add-ssh-user + variables: + - name: sshKey + required: false + schema: + openAPIV3Schema: + description: Public key to SSH onto the cluster nodes. + type: string + workers: + machineDeployments: + - class: ${CLUSTER_CLASS_NAME}-worker + machineHealthCheck: + maxUnhealthy: 40% + nodeStartupTimeout: 10m + unhealthyConditions: + - status: "False" + timeout: 300s + type: Ready + - status: Unknown + timeout: 300s + type: Ready + - status: "True" + timeout: 300s + type: MemoryPressure + - status: "True" + timeout: 300s + type: DiskPressure + - status: "True" + timeout: 300s + type: PIDPressure + - status: "True" + timeout: 300s + type: NetworkUnavailable + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-kcfg-0 + namespace: ${NAMESPACE} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: NutanixMachineTemplate + name: ${CLUSTER_CLASS_NAME}-md-nmt + namespace: ${NAMESPACE} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: ${CLUSTER_CLASS_NAME}-kcpt + namespace: ${NAMESPACE} +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + - 0.0.0.0 + extraArgs: + cloud-provider: external + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + controllerManager: + extraArgs: + cloud-provider: external + enable-hostpath-provisioner: "true" + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + scheduler: + extraArgs: + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + args: + - manager + env: + - name: vip_arp + value: "true" + - name: address + value: "${CONTROL_PLANE_ENDPOINT_IP}" + - name: port + value: "${CONTROL_PLANE_ENDPOINT_PORT=6443}" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: svc_enable + value: "${KUBEVIP_SVC_ENABLE=false}" + - name: lb_enable + value: "${KUBEVIP_LB_ENABLE=false}" + - name: enableServicesElection + value: "${KUBEVIP_SVC_ELECTION=false}" + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_TIME + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + resources: {} + hostNetwork: true + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + volumes: + - name: kubeconfig + hostPath: + type: FileOrCreate + path: /etc/kubernetes/admin.conf + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% + tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + postKubeadmCommands: + - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc + - echo "after kubeadm call" > /var/log/postkubeadm.log + preKubeadmCommands: + - echo "before kubeadm call" > /var/log/prekubeadm.log + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + useExperimentalRetryJoin: true + verbosity: 10 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: NutanixClusterTemplate +metadata: + name: ${CLUSTER_CLASS_NAME}-nct + namespace: ${NAMESPACE} +spec: + template: + spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_IP} + port: ${CONTROL_PLANE_ENDPOINT_PORT=6443} + failureDomains: [] + prismCentral: + additionalTrustBundle: + kind: ConfigMap + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + address: ${NUTANIX_ENDPOINT} + credentialRef: + kind: Secret + name: ${CLUSTER_NAME} + insecure: ${NUTANIX_INSECURE=false} + port: ${NUTANIX_PORT=9440} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: NutanixMachineTemplate +metadata: + name: ${CLUSTER_CLASS_NAME}-cp-nmt + namespace: ${NAMESPACE} +spec: + template: + spec: + bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} + cluster: + name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} + type: name + image: + name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} + type: name + memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} + providerID: nutanix://${CLUSTER_NAME}-m1 + subnet: + - name: ${NUTANIX_SUBNET_NAME} + type: name + systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} + vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} + vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: NutanixMachineTemplate +metadata: + name: ${CLUSTER_CLASS_NAME}-md-nmt + namespace: ${NAMESPACE} +spec: + template: + spec: + bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} + cluster: + name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} + type: name + image: + name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} + type: name + memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} + providerID: nutanix://${CLUSTER_NAME}-m1 + subnet: + - name: ${NUTANIX_SUBNET_NAME} + type: name + systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} + vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} + vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1}