Add node labels and Calico values

Tiltfile

@@ -380,8 +380,8 @@ def deploy_worker_templates(template, substitutions):
 
 def get_addons(flavor_name):
     # do not install calico and out of tree cloud provider for aks workload cluster
-    # if "aks" in flavor_name:
-    #     return ""
+    if "aks" in flavor_name:
+        return ""
 
     addon_cmd = ""
     if "intree-cloud-provider" not in flavor_name:

templates/addons/calico-aks/values.yaml

@@ -0,0 +1,9 @@
+installation:
+  kubernetesProvider: AKS
+  cni:
+    type: Calico
+  calicoNetwork:
+    bgp: Disabled
+    ipPools:
+    - cidr: 192.168.0.0/16
+      encapsulation: VXLAN

test/e2e/aks.go

@@ -55,7 +55,7 @@ func WaitForAKSControlPlaneInitialized(ctx context.Context, input clusterctl.App
 		Getter:  client,
 		Cluster: result.Cluster,
 	}, input.WaitForControlPlaneIntervals...)
-	InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, false)
+	InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Services.CIDRBlocks, false, true)
 }
 
 // WaitForAKSControlPlaneReady waits for the azure managed control plane to be ready.

test/e2e/aks_byo_node.go

@@ -167,11 +167,44 @@ func AKSBYONodeSpec(ctx context.Context, inputGetter func() AKSBYONodeSpecInput)
 		}, input.WaitIntervals...).Should(Succeed(), "Deleted KubeadmConfig %s/%s still exists", kubeadmConfig.Namespace, kubeadmConfig.Name)
 	}()
 
+	By("creating a Kubernetes client to the workload cluster")
+	workloadClusterProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Spec.ControlPlaneRef.Namespace, input.Cluster.Spec.ControlPlaneRef.Name)
+	Expect(workloadClusterProxy).NotTo(BeNil())
+	clientset := workloadClusterProxy.GetClientSet()
+	Expect(clientset).NotTo(BeNil())
+
+	By("Adding the expected AKS labels to the nodes")
+	// TODO: move this to the MachinePool object once MachinePools support label propagation
+	Eventually(func(g Gomega) {
+		nodeList, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+		g.Expect(err).NotTo(HaveOccurred())
+		for i, node := range nodeList.Items {
+			if _, ok := node.Labels["kubernetes.azure.com/cluster"]; !ok {
+				node.Labels["kubernetes.azure.com/cluster"] = infraControlPlane.Spec.NodeResourceGroupName
+				_, err := clientset.CoreV1().Nodes().Update(ctx, &nodeList.Items[i], metav1.UpdateOptions{})
+				g.Expect(err).NotTo(HaveOccurred())
+			}
+		}
+	}, input.WaitIntervals...).Should(Succeed())
+
+	// By("Deleting the kubeadm-config configmap")
+	// // TODO: this is a workaround for Calico Tigera-operator error about missing PodSubnets in the kubeadm-config configmap
+	// // There might be a better way to do this
+	// Eventually(func(g Gomega) {
+	// 	err := clientset.CoreV1().ConfigMaps("kube-system").Delete(ctx, "kubeadm-config", metav1.DeleteOptions{})
+	// 	g.Expect(err).NotTo(HaveOccurred())
+	// }, input.WaitIntervals...).Should(Succeed())
+
 	By("Verifying the MachinePool becomes ready")
 	Eventually(func(g Gomega) {
 		pool := &expv1.MachinePool{}
 		err := mgmtClient.Get(ctx, client.ObjectKeyFromObject(machinePool), pool)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(conditions.IsTrue(pool, clusterv1.ReadyCondition)).To(BeTrue())
 	}, input.WaitIntervals...).Should(Succeed())
+
+	// TODOs:
+	// - Add kubernetes.azure.com/cluster: MC_cecile-byo-aks-32693_cecile-byo-aks-32693_eastus label to install cloud-provider
+	// - Install Calico CNI with AKS values.yaml
+	// - remove kubeadm-config configmap in the cluster or add podSubnet to it
 }

test/e2e/cloud-provider-azure.go

@@ -68,7 +68,7 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus
 	InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "")
 
 	// We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized.
-	InstallCNI(ctx, input, cidrBlocks, hasWindows)
+	InstallCNI(ctx, input, cidrBlocks, hasWindows, false)
 
 	By("Waiting for Ready cloud-controller-manager deployment pods")
 	for _, d := range []string{"cloud-controller-manager"} {

test/e2e/cni.go

@@ -43,11 +43,11 @@ const (
 )
 
 // InstallCNI installs the CNI plugin depending on the input.CNIManifestPath
-func InstallCNI(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) {
+func InstallCNI(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool, isAKS bool) {
 	if input.CNIManifestPath != "" {
 		InstallCNIManifest(ctx, input, cidrBlocks, hasWindows)
 	} else {
-		InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows)
+		InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows, isAKS)
 	}
 }
 
@@ -64,11 +64,11 @@ func InstallCNIManifest(ctx context.Context, input clusterctl.ApplyCustomCluster
 
 // InstallCalicoHelmChart installs the official calico helm chart
 // and validates that expected pods exist and are Ready.
-func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) {
+func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool, isAKS bool) {
 	specName := "calico-install"
 
 	By("Installing Calico CNI via helm")
-	values := getCalicoValues(cidrBlocks)
+	values := getCalicoValues(cidrBlocks, isAKS)
 	clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName)
 	InstallHelmChart(ctx, clusterProxy, calicoOperatorNamespace, calicoHelmChartRepoURL, calicoHelmChartName, calicoHelmReleaseName, values, os.Getenv(CalicoVersion))
 	workloadClusterClient := clusterProxy.GetClient()
@@ -96,7 +96,7 @@ func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyCustomClu
 	}
 }
 
-func getCalicoValues(cidrBlocks []string) *HelmOptions {
+func getCalicoValues(cidrBlocks []string, isAKS bool) *HelmOptions {
 	var ipv6CidrBlock, ipv4CidrBlock string
 	var values *HelmOptions
 	for _, cidr := range cidrBlocks {
@@ -109,6 +109,12 @@ func getCalicoValues(cidrBlocks []string) *HelmOptions {
 	}
 	addonsPath := e2eConfig.GetVariable(AddonsPath)
 	switch {
+	case isAKS:
+		By("Configuring calico CNI helm chart for AKS configuration")
+		values = &HelmOptions{
+			StringValues: []string{fmt.Sprintf("installation.calicoNetwork.ipPools[0].cidr=%s", ipv4CidrBlock)},
+			ValueFiles:   []string{filepath.Join(addonsPath, "calico-aks", "values.yaml")},
+		}
 	case ipv6CidrBlock != "" && ipv4CidrBlock != "":
 		By("Configuring calico CNI helm chart for dual-stack configuration")
 		values = &HelmOptions{

test/e2e/common.go

@@ -279,7 +279,7 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu
 		// There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external.
 		InstallCalicoAndCloudProviderAzureHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows)
 	} else {
-		InstallCNI(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows)
+		InstallCNI(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows, false)
 	}
 	controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result)
 	InstallAzureDiskCSIDriverHelmChart(ctx, input, hasWindows)