diff --git a/Makefile b/Makefile index 1e63227a..d34baf1a 100644 --- a/Makefile +++ b/Makefile @@ -255,6 +255,7 @@ generate-e2e-templates: $(KUSTOMIZE) $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-managed --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-managed.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-managed-cluster-identity --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-managed-cluster-identity.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-cluster-identity --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-cluster-identity.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico.yaml .PHONY: test-e2e-run test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index b41bd64f..14407336 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -24,6 +24,7 @@ source "${REPO_ROOT}/hack/ensure-tags.sh" : "${OCI_UPGRADE_IMAGE_ID:?Environment variable empty or not defined.}" : "${OCI_ALTERNATIVE_REGION_IMAGE_ID:?Environment variable empty or not defined.}" : OCI_MANAGED_NODE_IMAGE_ID +: OCI_WINDOWS_IMAGE_ID export LOCAL_ONLY=${LOCAL_ONLY:-"true"} diff --git a/templates/cluster-template-windows-calico.yaml b/templates/cluster-template-windows-calico.yaml new file mode 100644 index 00000000..a036fd61 --- /dev/null +++ b/templates/cluster-template-windows-calico.yaml @@ -0,0 +1,156 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + cni: calico + csi-proxy: enabled + windows: enabled + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: OCIMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + apiServer: + certSANs: [localhost, 127.0.0.1] + extraArgs: + cloud-provider: oci + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_CONTROL_PLANE_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E4.Flex}" + shapeConfig: + ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + template: + spec: + imageId: "${OCI_NODE_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E4.Flex}" + shapeConfig: + ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}" + vnicAttachments: + - displayName: "CalicoNic" + nicIndex: 1 # second nic must be used for hyper-v + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds.meta_data["instance_id"] }} + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1 + - powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${NODE_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-win" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-win" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate \ No newline at end of file diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 1b442858..fb1a97f8 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -199,6 +199,29 @@ var _ = Describe("Workload cluster creation", func() { validateOLImage(namespace.Name, clusterName) }) + It("Windows - With 1 Linux control-plane nodes and with 1 Windows worker nodes using Calico CNI", func() { + clusterName = getClusterName(clusterNamePrefix, "windows-calico") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "windows-calico", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-windows-worker-nodes"), + }, result) + validateWindowsImage(namespace.Name, clusterName) + }) + It("Cloud Provider OCI testing [PRBlocking]", func() { clusterName = getClusterName(clusterNamePrefix, "ccm-testing") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -983,6 +1006,33 @@ func validateOLImage(nameSpace string, clusterName string) { } } +func validateWindowsImage(nameSpace string, clusterName string) { + lister := bootstrapClusterProxy.GetClient() + inClustersNamespaceListOption := client.InNamespace(nameSpace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterLabelName: clusterName, + } + + machineList := &clusterv1.MachineList{} + Expect(lister.List(context.Background(), machineList, inClustersNamespaceListOption, matchClusterListOption)). + To(Succeed(), "Couldn't list machines for the cluster %q", clusterName) + + Expect(len(machineList.Items)).To(Equal(2)) + for _, machine := range machineList.Items { + if machine.Labels["os"] == "windows" { + instanceOcid := strings.Split(*machine.Spec.ProviderID, "//")[1] + Log(fmt.Sprintf("Instance OCID is %s", instanceOcid)) + resp, err := computeClient.GetInstance(context.Background(), core.GetInstanceRequest{ + InstanceId: common.String(instanceOcid), + }) + Expect(err).NotTo(HaveOccurred()) + instanceSourceDetails, ok := resp.SourceDetails.(core.InstanceSourceViaImageDetails) + Expect(ok).To(BeTrue()) + Expect(*instanceSourceDetails.ImageId).To(Equal(os.Getenv("OCI_WINDOWS_IMAGE_ID"))) + } + } +} + func getClusterName(prefix, specName string) string { clusterName := os.Getenv("CLUSTER_NAME") if clusterName == "" { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 1e6193a4..f3300655 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -73,6 +73,7 @@ providers: - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-managed.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-managed-cluster-identity.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-cluster-identity.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-windows-calico.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/metadata.yaml" variables: @@ -111,6 +112,7 @@ intervals: default/wait-cluster: ["30m", "10s"] default/wait-control-plane: ["30m", "10s"] default/wait-worker-nodes: ["30m", "10s"] + default/wait-windows-worker-nodes: ["60m", "30s"] default/wait-cluster-bare-metal: [ "60m", "10s" ] default/wait-control-plane-bare-metal: [ "60m", "10s" ] default/wait-worker-nodes-bare-metal: [ "60m", "10s" ] diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml new file mode 100644 index 00000000..d1f72456 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/crs.yaml + - ../bases/ccm.yaml +patchesStrategicMerge: + - ./md.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml new file mode 100644 index 00000000..9f291624 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml @@ -0,0 +1,46 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + labels: + os: windows +spec: + template: + spec: + imageId: "${OCI_WINDOWS_IMAGE_ID}" + shape: "BM.Standard.E4.128" + shapeConfig: + ocpus: "128" + vnicAttachments: + - displayName: "CalicoNic" + nicIndex: 1 # second nic must be used for hyper-v + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=false} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + provider-id: oci://{{ ds.meta_data["instance_id"] }} + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1 + - powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + failureDomain: "2" \ No newline at end of file