From 0c4078110350c9c821bbb0d55e45d9ec295e2ead Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Wed, 6 Mar 2024 14:20:45 +0530 Subject: [PATCH] test/e2e: Add the e2e tests for flatcar-sysext Signed-off-by: Sayan Chowdhury Signed-off-by: Mathieu Tortuyaux --- .../cloud-provider-azure-flatcar-sysext.yaml | 18 + .../cluster-template-flatcar-sysext.yaml | 2 +- .../cluster-template-prow-flatcar-sysext.yaml | 405 ++++++++++++++++++ ...l-cloud-provider-azure-flatcar-sysext.yaml | 6 + .../ci/prow-flatcar-sysext/kustomization.yaml | 13 + test/e2e/azure_test.go | 38 ++ test/e2e/config/azure-dev.yaml | 2 + 7 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 templates/addons/cluster-api-helm/cloud-provider-azure-flatcar-sysext.yaml create mode 100644 templates/test/ci/cluster-template-prow-flatcar-sysext.yaml create mode 100644 templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar-sysext.yaml create mode 100644 templates/test/ci/prow-flatcar-sysext/kustomization.yaml diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar-sysext.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar-sysext.yaml new file mode 100644 index 00000000000..a5aa0f69e27 --- /dev/null +++ b/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar-sysext.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-flatcar-sysext +spec: + clusterSelector: + matchLabels: + cloud-provider: "azure-flatcar-sysext" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure-oot + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 + caCertDir: /usr/share/ca-certificates diff --git a/templates/cluster-template-flatcar-sysext.yaml b/templates/cluster-template-flatcar-sysext.yaml index c146e9b257f..c8e9adffc32 100644 --- a/templates/cluster-template-flatcar-sysext.yaml +++ b/templates/cluster-template-flatcar-sysext.yaml @@ -240,7 +240,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml new file mode 100644 index 00000000000..2d1088573da --- /dev/null +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -0,0 +1,405 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: |- + installation: + cni: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + registry: mcr.microsoft.com/oss + # Image and registry configuration for the tigera/operator pod. + tigeraOperator: + image: tigera/operator + registry: mcr.microsoft.com/oss + calicoctl: + image: mcr.microsoft.com/oss/calico/ctl + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-flatcar-sysext + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-flatcar-sysext + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 + caCertDir: /usr/share/ca-certificates +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: azure-flatcar-sysext + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + overwrite: false + partitions: [] + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: kubeadm.service + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + After=oem-cloudinit.service + # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. + After=containerd.service + # Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7679. + storage: + disks: + - device: /dev/disk/azure/scsi1/lun0 + partitions: + - number: 1 + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '@@HOSTNAME@@' + mounts: + - - etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + computeGallery: + gallery: flatcar-23485951-527a-48d6-9d11-6931ff0afc2e + name: flatcar-stable-amd64 + version: ${FLATCAR_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + computeGallery: + gallery: flatcar-23485951-527a-48d6-9d11-6931ff0afc2e + name: flatcar-stable-amd64 + version: ${FLATCAR_VERSION} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} diff --git a/templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar-sysext.yaml b/templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar-sysext.yaml new file mode 100644 index 00000000000..93472da223a --- /dev/null +++ b/templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar-sysext.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + cloud-provider: azure-flatcar-sysext diff --git a/templates/test/ci/prow-flatcar-sysext/kustomization.yaml b/templates/test/ci/prow-flatcar-sysext/kustomization.yaml new file mode 100644 index 00000000000..581093b33cf --- /dev/null +++ b/templates/test/ci/prow-flatcar-sysext/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../../flavors/flatcar-sysext/ + - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-flatcar-sysext.yaml +patchesStrategicMerge: + - ../patches/tags.yaml + - ../patches/controller-manager.yaml + - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure-flatcar-sysext.yaml diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 4496c5ee062..5d60eea4c3e 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -393,6 +393,44 @@ var _ = Describe("Workload cluster creation", func() { }) }) + Context("Creating a Flatcar sysext cluster [OPTIONAL]", func() { + It("With Flatcar control-plane and worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "flatcar-sysext") + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withFlavor("flatcar-sysext"), + withNamespace(namespace.Name), + withClusterName(clusterName), + withKubernetesVersion(e2eConfig.GetVariable(capi_e2e.KubernetesVersion)), + withControlPlaneMachineCount(1), + withWorkerMachineCount(1), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, + }), + withPostMachinesProvisioned(func() { + EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { + return DaemonsetsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }), + ), result) + + By("can create and access a load balancer", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + }) + }) + Context("Creating a cluster with spot vms [OPTIONAL]", func() { It("With spot vm machine deployments", func() { clusterName = getClusterName(clusterNamePrefix, "spot") diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index f3f67191912..b52caaa5aec 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -164,6 +164,8 @@ providers: targetName: "cluster-template-topology.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-flatcar.yaml" targetName: "cluster-template-flatcar.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml" + targetName: "cluster-template-flatcar-sysext.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-edgezone.yaml" targetName: "cluster-template-edgezone.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml"