Skip to content

Commit

Permalink
feat: add windows support
Browse files Browse the repository at this point in the history
  • Loading branch information
joekr committed Mar 8, 2023
1 parent c38f63e commit 72c5fb8
Show file tree
Hide file tree
Showing 7 changed files with 263 additions and 0 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ generate-e2e-templates: $(KUSTOMIZE)
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-managed --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-managed.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-managed-cluster-identity --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-managed-cluster-identity.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-cluster-identity --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-cluster-identity.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico.yaml

.PHONY: test-e2e-run
test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests
Expand Down
1 change: 1 addition & 0 deletions scripts/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ source "${REPO_ROOT}/hack/ensure-tags.sh"
: "${OCI_UPGRADE_IMAGE_ID:?Environment variable empty or not defined.}"
: "${OCI_ALTERNATIVE_REGION_IMAGE_ID:?Environment variable empty or not defined.}"
: OCI_MANAGED_NODE_IMAGE_ID
: OCI_WINDOWS_IMAGE_ID

export LOCAL_ONLY=${LOCAL_ONLY:-"true"}

Expand Down
156 changes: 156 additions & 0 deletions templates/cluster-template-windows-calico.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
labels:
cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}"
cni: calico
csi-proxy: enabled
windows: enabled
name: "${CLUSTER_NAME}"
namespace: "${NAMESPACE}"
spec:
clusterNetwork:
pods:
cidrBlocks:
- ${POD_CIDR:="192.168.0.0/16"}
serviceDomain: ${SERVICE_DOMAIN:="cluster.local"}
services:
cidrBlocks:
- ${SERVICE_CIDR:="10.128.0.0/12"}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: OCICluster
name: "${CLUSTER_NAME}"
namespace: "${NAMESPACE}"
controlPlaneRef:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
name: "${CLUSTER_NAME}-control-plane"
namespace: "${NAMESPACE}"
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: OCICluster
metadata:
labels:
cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}"
name: "${CLUSTER_NAME}"
spec:
compartmentId: "${OCI_COMPARTMENT_ID}"
---
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
metadata:
name: "${CLUSTER_NAME}-control-plane"
namespace: "${NAMESPACE}"
spec:
version: "${KUBERNETES_VERSION}"
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
machineTemplate:
infrastructureRef:
kind: OCIMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
namespace: "${NAMESPACE}"
kubeadmConfigSpec:
clusterConfiguration:
kubernetesVersion: ${KUBERNETES_VERSION}
apiServer:
certSANs: [localhost, 127.0.0.1]
extraArgs:
cloud-provider: oci
dns: {}
etcd: {}
networking: {}
scheduler: {}
initConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
kubeletExtraArgs:
cloud-provider: external
provider-id: oci://{{ ds["id"] }}
joinConfiguration:
discovery: {}
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
kubeletExtraArgs:
cloud-provider: external
provider-id: oci://{{ ds["id"] }}
---
kind: OCIMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec:
imageId: "${OCI_CONTROL_PLANE_IMAGE_ID}"
compartmentId: "${OCI_COMPARTMENT_ID}"
shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E4.Flex}"
shapeConfig:
ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}"
metadata:
ssh_authorized_keys: "${OCI_SSH_KEY}"
isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: OCIMachineTemplate
metadata:
name: "${CLUSTER_NAME}-md-win"
spec:
template:
spec:
imageId: "${OCI_NODE_IMAGE_ID}"
compartmentId: "${OCI_COMPARTMENT_ID}"
shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E4.Flex}"
shapeConfig:
ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}"
vnicAttachments:
- displayName: "CalicoNic"
nicIndex: 1 # second nic must be used for hyper-v
metadata:
ssh_authorized_keys: "${OCI_SSH_KEY}"
isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
kind: KubeadmConfigTemplate
metadata:
name: "${CLUSTER_NAME}-md-win"
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
criSocket: npipe:////./pipe/containerd-containerd
kubeletExtraArgs:
cloud-provider: external
provider-id: oci://{{ ds.meta_data["instance_id"] }}
feature-gates: WindowsHostProcessContainers=true
v: "2"
windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands:
- powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1
- powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
name: "${CLUSTER_NAME}-md-win"
spec:
clusterName: "${CLUSTER_NAME}"
replicas: ${NODE_MACHINE_COUNT}
selector:
matchLabels:
template:
spec:
clusterName: "${CLUSTER_NAME}"
version: "${KUBERNETES_VERSION}"
bootstrap:
configRef:
name: "${CLUSTER_NAME}-md-win"
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
infrastructureRef:
name: "${CLUSTER_NAME}-md-win"
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: OCIMachineTemplate
50 changes: 50 additions & 0 deletions test/e2e/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,29 @@ var _ = Describe("Workload cluster creation", func() {
validateOLImage(namespace.Name, clusterName)
})

It("Windows - With 1 Linux control-plane nodes and with 1 Windows worker nodes using Calico CNI", func() {
clusterName = getClusterName(clusterNamePrefix, "windows-calico")
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: bootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()),
ClusterctlConfigPath: clusterctlConfigPath,
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: "windows-calico",
Namespace: namespace.Name,
ClusterName: clusterName,
KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-windows-worker-nodes"),
}, result)
validateWindowsImage(namespace.Name, clusterName)
})

It("Cloud Provider OCI testing [PRBlocking]", func() {
clusterName = getClusterName(clusterNamePrefix, "ccm-testing")
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
Expand Down Expand Up @@ -983,6 +1006,33 @@ func validateOLImage(nameSpace string, clusterName string) {
}
}

func validateWindowsImage(nameSpace string, clusterName string) {
lister := bootstrapClusterProxy.GetClient()
inClustersNamespaceListOption := client.InNamespace(nameSpace)
matchClusterListOption := client.MatchingLabels{
clusterv1.ClusterLabelName: clusterName,
}

machineList := &clusterv1.MachineList{}
Expect(lister.List(context.Background(), machineList, inClustersNamespaceListOption, matchClusterListOption)).
To(Succeed(), "Couldn't list machines for the cluster %q", clusterName)

Expect(len(machineList.Items)).To(Equal(2))
for _, machine := range machineList.Items {
if machine.Labels["os"] == "windows" {
instanceOcid := strings.Split(*machine.Spec.ProviderID, "//")[1]
Log(fmt.Sprintf("Instance OCID is %s", instanceOcid))
resp, err := computeClient.GetInstance(context.Background(), core.GetInstanceRequest{
InstanceId: common.String(instanceOcid),
})
Expect(err).NotTo(HaveOccurred())
instanceSourceDetails, ok := resp.SourceDetails.(core.InstanceSourceViaImageDetails)
Expect(ok).To(BeTrue())
Expect(*instanceSourceDetails.ImageId).To(Equal(os.Getenv("OCI_WINDOWS_IMAGE_ID")))
}
}
}

func getClusterName(prefix, specName string) string {
clusterName := os.Getenv("CLUSTER_NAME")
if clusterName == "" {
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/config/e2e_conf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ providers:
- sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-managed.yaml"
- sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-managed-cluster-identity.yaml"
- sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-cluster-identity.yaml"
- sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-windows-calico.yaml"
- sourcePath: "../data/infrastructure-oci/v1beta1/metadata.yaml"

variables:
Expand Down Expand Up @@ -111,6 +112,7 @@ intervals:
default/wait-cluster: ["30m", "10s"]
default/wait-control-plane: ["30m", "10s"]
default/wait-worker-nodes: ["30m", "10s"]
default/wait-windows-worker-nodes: ["60m", "30s"]
default/wait-cluster-bare-metal: [ "60m", "10s" ]
default/wait-control-plane-bare-metal: [ "60m", "10s" ]
default/wait-worker-nodes-bare-metal: [ "60m", "10s" ]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bases:
- ../bases/cluster.yaml
- ../bases/md.yaml
- ../bases/crs.yaml
- ../bases/ccm.yaml
patchesStrategicMerge:
- ./md.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: OCIMachineTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
labels:
os: windows
spec:
template:
spec:
imageId: "${OCI_WINDOWS_IMAGE_ID}"
shape: "BM.Standard.E4.128"
shapeConfig:
ocpus: "128"
vnicAttachments:
- displayName: "CalicoNic"
nicIndex: 1 # second nic must be used for hyper-v
isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=false}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
kind: KubeadmConfigTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
criSocket: npipe:////./pipe/containerd-containerd
kubeletExtraArgs:
provider-id: oci://{{ ds.meta_data["instance_id"] }}
feature-gates: WindowsHostProcessContainers=true
v: "2"
windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands:
- powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1
- powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
failureDomain: "2"

0 comments on commit 72c5fb8

Please sign in to comment.