From bfc487bebd6d1f7c7b18071bd0ea8d94cc14505f Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Fri, 6 Oct 2023 00:38:43 +0000 Subject: [PATCH] Add node labels and Calico values --- Tiltfile | 4 +-- templates/addons/calico-aks/values.yaml | 9 +++++++ test/e2e/aks.go | 2 +- test/e2e/aks_byo_node.go | 33 +++++++++++++++++++++++++ test/e2e/cloud-provider-azure.go | 2 +- test/e2e/cni.go | 16 ++++++++---- test/e2e/common.go | 2 +- 7 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 templates/addons/calico-aks/values.yaml diff --git a/Tiltfile b/Tiltfile index 67417999f696..fc284c0a6560 100644 --- a/Tiltfile +++ b/Tiltfile @@ -380,8 +380,8 @@ def deploy_worker_templates(template, substitutions): def get_addons(flavor_name): # do not install calico and out of tree cloud provider for aks workload cluster - # if "aks" in flavor_name: - # return "" + if "aks" in flavor_name: + return "" addon_cmd = "" if "intree-cloud-provider" not in flavor_name: diff --git a/templates/addons/calico-aks/values.yaml b/templates/addons/calico-aks/values.yaml new file mode 100644 index 000000000000..06168ed2bee7 --- /dev/null +++ b/templates/addons/calico-aks/values.yaml @@ -0,0 +1,9 @@ +installation: + kubernetesProvider: AKS + cni: + type: Calico + calicoNetwork: + bgp: Disabled + ipPools: + - cidr: 192.168.0.0/16 + encapsulation: VXLAN diff --git a/test/e2e/aks.go b/test/e2e/aks.go index d9aabb5816ea..c2015a8e4489 100644 --- a/test/e2e/aks.go +++ b/test/e2e/aks.go @@ -55,7 +55,7 @@ func WaitForAKSControlPlaneInitialized(ctx context.Context, input clusterctl.App Getter: client, Cluster: result.Cluster, }, input.WaitForControlPlaneIntervals...) - InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, false) + InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, false, true) } // WaitForAKSControlPlaneReady waits for the azure managed control plane to be ready. diff --git a/test/e2e/aks_byo_node.go b/test/e2e/aks_byo_node.go index ed643d0300dc..039b9a3fbec5 100644 --- a/test/e2e/aks_byo_node.go +++ b/test/e2e/aks_byo_node.go @@ -167,6 +167,34 @@ func AKSBYONodeSpec(ctx context.Context, inputGetter func() AKSBYONodeSpecInput) }, input.WaitIntervals...).Should(Succeed(), "Deleted KubeadmConfig %s/%s still exists", kubeadmConfig.Namespace, kubeadmConfig.Name) }() + By("creating a Kubernetes client to the workload cluster") + workloadClusterProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Spec.ControlPlaneRef.Namespace, input.Cluster.Spec.ControlPlaneRef.Name) + Expect(workloadClusterProxy).NotTo(BeNil()) + clientset := workloadClusterProxy.GetClientSet() + Expect(clientset).NotTo(BeNil()) + + By("Adding the expected AKS labels to the nodes") + // TODO: move this to the MachinePool object once MachinePools support label propagation + Eventually(func(g Gomega) { + nodeList, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + for i, node := range nodeList.Items { + if _, ok := node.Labels["kubernetes.azure.com/cluster"]; !ok { + node.Labels["kubernetes.azure.com/cluster"] = infraControlPlane.Spec.NodeResourceGroupName + _, err := clientset.CoreV1().Nodes().Update(ctx, &nodeList.Items[i], metav1.UpdateOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + } + } + }, input.WaitIntervals...).Should(Succeed()) + + // By("Deleting the kubeadm-config configmap") + // // TODO: this is a workaround for Calico Tigera-operator error about missing PodSubnets in the kubeadm-config configmap + // // There might be a better way to do this + // Eventually(func(g Gomega) { + // err := clientset.CoreV1().ConfigMaps("kube-system").Delete(ctx, "kubeadm-config", metav1.DeleteOptions{}) + // g.Expect(err).NotTo(HaveOccurred()) + // }, input.WaitIntervals...).Should(Succeed()) + By("Verifying the MachinePool becomes ready") Eventually(func(g Gomega) { pool := &expv1.MachinePool{} @@ -174,4 +202,9 @@ func AKSBYONodeSpec(ctx context.Context, inputGetter func() AKSBYONodeSpecInput) g.Expect(err).NotTo(HaveOccurred()) g.Expect(conditions.IsTrue(pool, clusterv1.ReadyCondition)).To(BeTrue()) }, input.WaitIntervals...).Should(Succeed()) + + // TODOs: + // - Add kubernetes.azure.com/cluster: MC_cecile-byo-aks-32693_cecile-byo-aks-32693_eastus label to install cloud-provider + // - Install Calico CNI with AKS values.yaml + // - remove kubeadm-config configmap in the cluster or add podSubnet to it } diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 34848f068170..1b7787688de5 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -68,7 +68,7 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "") // We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. - InstallCNI(ctx, input, cidrBlocks, hasWindows) + InstallCNI(ctx, input, cidrBlocks, hasWindows, false) By("Waiting for Ready cloud-controller-manager deployment pods") for _, d := range []string{"cloud-controller-manager"} { diff --git a/test/e2e/cni.go b/test/e2e/cni.go index d4c5c5c7f292..997bc85cf276 100644 --- a/test/e2e/cni.go +++ b/test/e2e/cni.go @@ -43,11 +43,11 @@ const ( ) // InstallCNI installs the CNI plugin depending on the input.CNIManifestPath -func InstallCNI(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { +func InstallCNI(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool, isAKS bool) { if input.CNIManifestPath != "" { InstallCNIManifest(ctx, input, cidrBlocks, hasWindows) } else { - InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows) + InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows, isAKS) } } @@ -64,11 +64,11 @@ func InstallCNIManifest(ctx context.Context, input clusterctl.ApplyCustomCluster // InstallCalicoHelmChart installs the official calico helm chart // and validates that expected pods exist and are Ready. -func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { +func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool, isAKS bool) { specName := "calico-install" By("Installing Calico CNI via helm") - values := getCalicoValues(cidrBlocks) + values := getCalicoValues(cidrBlocks, isAKS) clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) InstallHelmChart(ctx, clusterProxy, calicoOperatorNamespace, calicoHelmChartRepoURL, calicoHelmChartName, calicoHelmReleaseName, values, os.Getenv(CalicoVersion)) workloadClusterClient := clusterProxy.GetClient() @@ -96,7 +96,7 @@ func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClu } } -func getCalicoValues(cidrBlocks []string) *HelmOptions { +func getCalicoValues(cidrBlocks []string, isAKS bool) *HelmOptions { var ipv6CidrBlock, ipv4CidrBlock string var values *HelmOptions for _, cidr := range cidrBlocks { @@ -109,6 +109,12 @@ func getCalicoValues(cidrBlocks []string) *HelmOptions { } addonsPath := e2eConfig.GetVariable(AddonsPath) switch { + case isAKS: + By("Configuring calico CNI helm chart for AKS configuration") + values = &HelmOptions{ + StringValues: []string{fmt.Sprintf("installation.calicoNetwork.ipPools[0].cidr=%s", ipv4CidrBlock)}, + ValueFiles: []string{filepath.Join(addonsPath, "calico-aks", "values.yaml")}, + } case ipv6CidrBlock != "" && ipv4CidrBlock != "": By("Configuring calico CNI helm chart for dual-stack configuration") values = &HelmOptions{ diff --git a/test/e2e/common.go b/test/e2e/common.go index eea819490d00..1a9de2c34005 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -279,7 +279,7 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. InstallCalicoAndCloudProviderAzureHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } else { - InstallCNI(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + InstallCNI(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows, false) } controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result) InstallAzureDiskCSIDriverHelmChart(ctx, input, hasWindows)