added first jsonPatches in clusterclass template and cluster topology

deepakm-ntnx · Dec 19, 2023 · e188070 · e188070
1 parent 9277af2
commit e188070
Show file tree

Hide file tree

Showing 29 changed files with 796 additions and 74 deletions.
diff --git a/Makefile b/Makefile
@@ -177,6 +177,8 @@ FLAVOR ?= e2e
 
 TEST_NAMESPACE=capx-test-ns
 TEST_CLUSTER_NAME=mycluster
+TEST_CLUSTER_CLASS_NAME=my-clusterclass
+TEST_TOPOLOGY_CLUSTER_NAME=my-cc-cluster
 
 # set ginkgo focus flags, if any
 ifneq ($(strip $(GINKGO_FOCUS)),)
@@ -323,6 +325,7 @@ cluster-e2e-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml
+	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml
 
 cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates without kubeproxy
 	# v1alpha4
@@ -342,6 +345,7 @@ cluster-e2e-templates-no-kubeproxy: $(KUSTOMIZE) ##Generate cluster templates wi
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml
 	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml
+	$(KUSTOMIZE) build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml
 
 cluster-templates: $(KUSTOMIZE) ## Generate cluster templates for all flavors
 	$(KUSTOMIZE) build $(TEMPLATES_DIR)/base > $(TEMPLATES_DIR)/cluster-template.yaml
@@ -415,32 +419,37 @@ list-workload-resources: ## Run kubectl queries to get all capx workload related
 
 .PHONY: test-cc-cluster-create
 test-cc-cluster-create: cluster-templates
-	clusterctl generate cluster cc-test --from ./templates/cluster-template-clusterclass.yaml -n $(TEST_NAMESPACE) > cc-test.yaml
-	clusterctl generate cluster cluster-topology --from ./templates/cluster-template-topology.yaml -n $(TEST_NAMESPACE) > cluster-topology.yaml
+	clusterctl generate cluster ${TEST_CLUSTER_CLASS_NAME} --from ./templates/cluster-template-clusterclass.yaml -n $(TEST_NAMESPACE) > ${TEST_CLUSTER_CLASS_NAME}.yaml
+	clusterctl generate cluster ${TEST_TOPOLOGY_CLUSTER_NAME} --from ./templates/cluster-template-topology.yaml -n $(TEST_NAMESPACE) > ${TEST_TOPOLOGY_CLUSTER_NAME}.yaml
 	kubectl create ns $(TEST_NAMESPACE) --dry-run=client -oyaml | kubectl apply --server-side -f -
-	kubectl apply --server-side -f ./cc-test.yaml
-	kubectl apply --server-side -f ./cluster-topology.yaml
+	kubectl apply --server-side -f ./${TEST_CLUSTER_CLASS_NAME}.yaml
+	kubectl apply --server-side -f ./${TEST_TOPOLOGY_CLUSTER_NAME}.yaml
 
 .PHONY: test-cc-cluster-delete
 test-cc-cluster-delete:
-	kubectl -n $(TEST_NAMESPACE) delete cluster cluster-topology --ignore-not-found
-	kubectl -n $(TEST_NAMESPACE) delete secret cluster-topology --ignore-not-found
-	kubectl -n $(TEST_NAMESPACE) delete cm user-ca-bundle --ignore-not-found
-	rm cluster-topology.yaml || true
-	rm cc-test.yaml || true
+	kubectl -n $(TEST_NAMESPACE) delete cluster ${TEST_TOPOLOGY_CLUSTER_NAME} --ignore-not-found
+	kubectl -n $(TEST_NAMESPACE) delete secret ${TEST_TOPOLOGY_CLUSTER_NAME} --ignore-not-found
+	kubectl -n $(TEST_NAMESPACE) delete cm ${TEST_TOPOLOGY_CLUSTER_NAME}-pc-trusted-ca-bundle --ignore-not-found
+	rm ${TEST_TOPOLOGY_CLUSTER_NAME}.yaml || true
+	rm ${TEST_CLUSTER_CLASS_NAME}.yaml || true
 
+.PHONY: generate-cc-cluster-kubeconfig
+generate-cc-cluster-kubeconfig:
+	kubectl -n ${TEST_NAMESPACE} get secret ${TEST_TOPOLOGY_CLUSTER_NAME}-kubeconfig -o json | jq -r .data.value | base64 --decode > ${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig
+
+.PHONY: test-cc-cluster-install-cni
+test-cc-cluster-install-cni: generate-cc-cluster-kubeconfig
+	kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig apply -f https://raw.githubusercontent.com/nutanix-cloud-native/cluster-api-provider-nutanix/main/test/e2e/data/cni/calico/calico.yaml
 
 .PHONY: list-cc-cluster-resources
-list-cc-cluster-resources:
+list-cc-cluster-resources: generate-cc-cluster-kubeconfig
 	kubectl -n capx-system get endpoints
 	kubectl get crd | grep nutanix
 	kubectl get cluster-api -A
 	kubectl -n $(TEST_NAMESPACE) get Cluster,NutanixCluster,Machine,NutanixMachine,KubeAdmControlPlane,MachineHealthCheck,nodes
 	kubectl get ValidatingWebhookConfiguration,MutatingWebhookConfiguration -A
-	kubectl -n ${TEST_NAMESPACE} get secret cluster-topology-kubeconfig -o json | jq -r .data.value | base64 --decode > cluster-topology.workload.kubeconfig
-	kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get nodes,ns
-	kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get nodes,ns
-	kubectl --kubeconfig ./cluster-topology.workload.kubeconfig get pods -A
+	kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig get nodes,ns
+	kubectl --kubeconfig ./${TEST_TOPOLOGY_CLUSTER_NAME}.workload.kubeconfig get pods -A
 
 .PHONY: ginkgo-help
 ginkgo-help:

diff --git a/templates/base-root/ccm-patch.yaml b/templates/base-root/ccm-patch.yaml
diff --git a/templates/base-root/kustomization.yaml b/templates/base-root/kustomization.yaml
@@ -12,9 +12,5 @@ configMapGenerator:
 bases:
   - ./secret.yaml
   - ./cm.yaml
-  - ./kct.yaml
   - ./nutanix-ccm-crs.yaml
   - ./nutanix-ccm-secret.yaml
-
-patchesStrategicMerge:
-- ./ccm-patch.yaml
diff --git a/templates/base-root/nutanix-ccm-crs.yaml b/templates/base-root/nutanix-ccm-crs.yaml
@@ -12,7 +12,7 @@ spec:
   - kind: Secret
     name: nutanix-ccm-secret
   - kind: ConfigMap
-    name: user-ca-bundle
+    name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
   strategy: ApplyOnce
 ---
 apiVersion: v1

diff --git a/templates/base/ccm-patch.yaml b/templates/base/ccm-patch.yaml
@@ -25,4 +25,16 @@ kind: Cluster
 metadata:
   labels:
     ccm: "nutanix"
-  name: "${CLUSTER_NAME}"
+  name: "${CLUSTER_NAME}"
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-kcfg-0"
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+            cloud-provider: external
diff --git a/templates/base-root/kct.yaml → templates/base/kct.yaml b/templates/base-root/kct.yaml → templates/base/kct.yaml
diff --git a/templates/base/kustomization.yaml b/templates/base/kustomization.yaml
@@ -8,6 +8,7 @@ bases:
   - ./nutanix-cluster.yaml
   - ./cluster-without-topology.yaml
   - ./kcp.yaml
+  - ./kct.yaml
   - ./nmt.yaml
   - ./md.yaml
   - ./mhc.yaml

diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml
@@ -254,7 +254,7 @@ spec:
   - kind: Secret
     name: nutanix-ccm-secret
   - kind: ConfigMap
-    name: user-ca-bundle
+    name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
   strategy: ApplyOnce
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
@@ -276,12 +276,6 @@ spec:
       preKubeadmCommands:
       - echo "before kubeadm call" > /var/log/prekubeadm.log
       - hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
-      users:
-      - lockPassword: false
-        name: capiuser
-        sshAuthorizedKeys:
-        - ${NUTANIX_SSH_AUTHORIZED_KEY}
-        sudo: ALL=(ALL) NOPASSWD:ALL
       verbosity: 10
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -330,9 +324,51 @@ spec:
       kind: NutanixClusterTemplate
       name: ${CLUSTER_CLASS_NAME}-nct
       namespace: ${NAMESPACE}
+  patches:
+  - definitions:
+    - jsonPatches:
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/users
+        valueFrom:
+          template: |
+            - name: capxuser
+              lockPassword: false
+              sudo: ALL=(ALL) NOPASSWD:ALL
+              sshAuthorizedKeys:
+                - '{{ .sshKey }}'
+      selector:
+        apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+        kind: KubeadmControlPlaneTemplate
+        matchResources:
+          controlPlane: true
+    - jsonPatches:
+      - op: add
+        path: /spec/template/spec/users
+        valueFrom:
+          template: |
+            - name: capxuser
+              lockPassword: false
+              sudo: ALL=(ALL) NOPASSWD:ALL
+              sshAuthorizedKeys:
+                - '{{ .sshKey }}'
+      selector:
+        apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+        kind: KubeadmConfigTemplate
+        matchResources:
+          machineDeploymentClass:
+            names:
+            - ${CLUSTER_CLASS_NAME}-worker
+    name: add-ssh-user
+  variables:
+  - name: sshKey
+    required: false
+    schema:
+      openAPIV3Schema:
+        description: Public key to SSH onto the cluster nodes.
+        type: string
   workers:
     machineDeployments:
-    - class: default-worker
+    - class: ${CLUSTER_CLASS_NAME}-worker
       machineHealthCheck:
         maxUnhealthy: 40%
         nodeStartupTimeout: 10m
@@ -480,12 +516,6 @@ spec:
         - echo "before kubeadm call" > /var/log/prekubeadm.log
         - hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
         useExperimentalRetryJoin: true
-        users:
-        - lockPassword: false
-          name: capiuser
-          sshAuthorizedKeys:
-          - ${NUTANIX_SSH_AUTHORIZED_KEY}
-          sudo: ALL=(ALL) NOPASSWD:ALL
         verbosity: 10
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

diff --git a/templates/cluster-template-csi.yaml b/templates/cluster-template-csi.yaml
@@ -1573,7 +1573,7 @@ spec:
   - kind: Secret
     name: nutanix-ccm-secret
   - kind: ConfigMap
-    name: user-ca-bundle
+    name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
   strategy: ApplyOnce
 ---
 apiVersion: addons.cluster.x-k8s.io/v1beta1

diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml
@@ -20,10 +20,13 @@ spec:
     controlPlane:
       metadata: {}
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+    variables:
+    - name: sshKey
+      value: ${NUTANIX_SSH_AUTHORIZED_KEY}
     version: ${KUBERNETES_VERSION}
     workers:
       machineDeployments:
-      - class: default-worker
+      - class: ${CLUSTER_CLASS_NAME}-worker
         metadata: {}
         name: md-0
         replicas: ${WORKER_MACHINE_COUNT}
diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -254,7 +254,7 @@ spec:
   - kind: Secret
     name: nutanix-ccm-secret
   - kind: ConfigMap
-    name: user-ca-bundle
+    name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
   strategy: ApplyOnce
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1

diff --git a/templates/clusterclass/clusterclass.yaml b/templates/clusterclass/clusterclass.yaml
@@ -39,7 +39,7 @@ spec:
           timeout: 300s
   workers:
     machineDeployments:
-    - class: default-worker
+    - class: ${CLUSTER_CLASS_NAME}-worker
       template:
         bootstrap:
           ref:
@@ -80,4 +80,46 @@ spec:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
       kind: NutanixClusterTemplate
       name: ${CLUSTER_CLASS_NAME}-nct
-      namespace: ${NAMESPACE}
+      namespace: ${NAMESPACE}
+  patches:
+  - definitions:
+      - jsonPatches:
+        - op: add
+          path: /spec/template/spec/kubeadmConfigSpec/users
+          valueFrom:
+            template: |
+              - name: capxuser
+                lockPassword: false
+                sudo: ALL=(ALL) NOPASSWD:ALL
+                sshAuthorizedKeys:
+                  - '{{ .sshKey }}'
+        selector:
+          apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+          kind: KubeadmControlPlaneTemplate
+          matchResources:
+            controlPlane: true
+      - jsonPatches:
+        - op: add
+          path: /spec/template/spec/users
+          valueFrom:
+            template: |
+              - name: capxuser
+                lockPassword: false
+                sudo: ALL=(ALL) NOPASSWD:ALL
+                sshAuthorizedKeys:
+                  - '{{ .sshKey }}'
+        selector:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          matchResources:
+            machineDeploymentClass:
+              names:
+              - ${CLUSTER_CLASS_NAME}-worker
+    name: add-ssh-user
+  variables:
+  - name: sshKey
+    required: false
+    schema:
+      openAPIV3Schema:
+        description: Public key to SSH onto the cluster nodes.
+        type: string
diff --git a/templates/clusterclass/kcpt.yaml b/templates/clusterclass/kcpt.yaml
@@ -108,10 +108,4 @@ spec:
           - echo "before kubeadm call" > /var/log/prekubeadm.log
           - hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
         useExperimentalRetryJoin: true
-        users:
-          - name: capiuser
-            lockPassword: false
-            sudo: ALL=(ALL) NOPASSWD:ALL
-            sshAuthorizedKeys:
-              - ${NUTANIX_SSH_AUTHORIZED_KEY}
         verbosity: 10
diff --git a/templates/clusterclass/kct.yaml b/templates/clusterclass/kct.yaml
@@ -0,0 +1,19 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-kcfg-0"
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+            cloud-provider: external
+            eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
+            tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256}
+      postKubeadmCommands:
+      - echo "after kubeadm call" > /var/log/postkubeadm.log
+      preKubeadmCommands:
+      - echo "before kubeadm call" > /var/log/prekubeadm.log
+      - hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
+      verbosity: 10
diff --git a/templates/clusterclass/kustomization.yaml b/templates/clusterclass/kustomization.yaml
@@ -9,4 +9,5 @@ bases:
   - ./clusterclass.yaml
   - ./nmt-cp.yaml
   - ./nmt-md.yaml
-  - ./kcpt.yaml
+  - ./kcpt.yaml
+  - ./kct.yaml
diff --git a/templates/topology/cluster-with-topology.yaml b/templates/topology/cluster-with-topology.yaml
@@ -19,7 +19,10 @@ spec:
       metadata: {}
     workers:
       machineDeployments:
-      - class: default-worker
+      - class: ${CLUSTER_CLASS_NAME}-worker
         name: md-0
         replicas: ${WORKER_MACHINE_COUNT}
-        metadata: {}
+        metadata: {}
+    variables:
+    - name: sshKey
+      value: '${NUTANIX_SSH_AUTHORIZED_KEY}'
diff --git a/test/e2e/data/infrastructure-nutanix/ccm-update.yaml b/test/e2e/data/infrastructure-nutanix/ccm-update.yaml
@@ -5,7 +5,7 @@ data:
     apiVersion: v1
     kind: ConfigMap
     metadata:
-      name: user-ca-bundle
+      name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
       namespace: kube-system
     binaryData:
       ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""}
@@ -37,7 +37,7 @@ data:
             },
             "additionalTrustBundle": {
               "kind": "ConfigMap",
-              "name": "user-ca-bundle",
+              "name": "${CLUSTER_NAME}-pc-trusted-ca-bundle",
               "namespace": "kube-system"
             }
           },
@@ -245,5 +245,5 @@ spec:
   - kind: Secret
     name: nutanix-ccm-secret
   - kind: ConfigMap
-    name: user-ca-bundle
+    name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
   strategy: ApplyOnce