From d8e7ec80b8488a5612d052b86750b5acc17fdf32 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Mon, 23 Jan 2023 12:20:48 -0500 Subject: [PATCH] Add e2e test --- Makefile | 1 + cloud/scope/machine.go | 1 - cloud/scope/vnic_reconciler.go | 4 +- .../compute/mock_compute/client_mock.go | 15 ++++++ cloud/services/vcn/mock_vcn/client_mock.go | 15 ++++++ .../cluster-template-windows-calico.yaml | 3 +- test/e2e/cluster_test.go | 50 +++++++++++++++++++ test/e2e/config/e2e_conf.yaml | 2 + .../kustomization.yaml | 7 +++ .../cluster-template-windows-calico/md.yaml | 46 +++++++++++++++++ 10 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml diff --git a/Makefile b/Makefile index 6b17a9a81..c10803662 100644 --- a/Makefile +++ b/Makefile @@ -252,6 +252,7 @@ generate-e2e-templates: $(KUSTOMIZE) $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-remote-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-remote-vcn-peering.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-externally-managed-vcn --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-externally-managed-vcn.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-machine-pool --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-machine-pool.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico.yaml .PHONY: test-e2e-run test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 75a2b5e0f..57d260935 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -624,7 +624,6 @@ func (m *MachineScope) getGetControlPlaneMachineNSGs() []string { // and returns the subnet ID if the name matches func (m *MachineScope) getMachineSubnet(name string) (*string, error) { for _, subnet := range m.OCICluster.Spec.NetworkSpec.Vcn.Subnets { - // if a subnet name is defined, use the correct subnet if subnet.Name == name { return subnet.ID, nil } diff --git a/cloud/scope/vnic_reconciler.go b/cloud/scope/vnic_reconciler.go index fb5b916d9..8a415fd42 100644 --- a/cloud/scope/vnic_reconciler.go +++ b/cloud/scope/vnic_reconciler.go @@ -19,7 +19,6 @@ func (m *MachineScope) ReconcileVnicAttachments(ctx context.Context) error { for index, vnicAttachment := range m.OCIMachine.Spec.VnicAttachments { if m.vnicAttachmentExists(ctx, vnicAttachment) { m.Logger.Info("vnicAttachment", vnicAttachment.DisplayName, " already exists") - // TODO: update vs create https://docs.oracle.com/en-us/iaas/api/#/en/iaas/20160918/Vnic/UpdateVnic m.updateVnicAttachment(ctx, vnicAttachment) continue } @@ -31,9 +30,8 @@ func (m *MachineScope) ReconcileVnicAttachments(ctx context.Context) error { m.Logger.Error(err, msg) return err } - // TODO: kick off the "flush" so ids are stored as soon as they are attached + m.OCIMachine.Spec.VnicAttachments[index].VnicAttachmentId = vnicId - vnicAttachment.VnicAttachmentId = common.String("TESTING") } return nil diff --git a/cloud/services/compute/mock_compute/client_mock.go b/cloud/services/compute/mock_compute/client_mock.go index 9a3987d09..56ad89dfa 100644 --- a/cloud/services/compute/mock_compute/client_mock.go +++ b/cloud/services/compute/mock_compute/client_mock.go @@ -35,6 +35,21 @@ func (m *MockComputeClient) EXPECT() *MockComputeClientMockRecorder { return m.recorder } +// AttachVnic mocks base method. +func (m *MockComputeClient) AttachVnic(ctx context.Context, request core.AttachVnicRequest) (core.AttachVnicResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AttachVnic", ctx, request) + ret0, _ := ret[0].(core.AttachVnicResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AttachVnic indicates an expected call of AttachVnic. +func (mr *MockComputeClientMockRecorder) AttachVnic(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVnic", reflect.TypeOf((*MockComputeClient)(nil).AttachVnic), ctx, request) +} + // GetInstance mocks base method. func (m *MockComputeClient) GetInstance(ctx context.Context, request core.GetInstanceRequest) (core.GetInstanceResponse, error) { m.ctrl.T.Helper() diff --git a/cloud/services/vcn/mock_vcn/client_mock.go b/cloud/services/vcn/mock_vcn/client_mock.go index e80e7ff6c..eeb3fc219 100644 --- a/cloud/services/vcn/mock_vcn/client_mock.go +++ b/cloud/services/vcn/mock_vcn/client_mock.go @@ -575,6 +575,21 @@ func (mr *MockClientMockRecorder) GetVnic(ctx, request interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVnic", reflect.TypeOf((*MockClient)(nil).GetVnic), ctx, request) } +// UpdateVnic mocks base method. +func (m *MockClient) UpdateVnic(ctx context.Context, request core.UpdateVnicRequest) (core.UpdateVnicResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateVnic", ctx, request) + ret0, _ := ret[0].(core.UpdateVnicResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateVnic indicates an expected call of UpdateVnic. +func (mr *MockClientMockRecorder) UpdateVnic(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVnic", reflect.TypeOf((*MockClient)(nil).UpdateVnic), ctx, request) +} + // ListDrgAttachments mocks base method. func (m *MockClient) ListDrgAttachments(ctx context.Context, request core.ListDrgAttachmentsRequest) (core.ListDrgAttachmentsResponse, error) { m.ctrl.T.Helper() diff --git a/templates/cluster-template-windows-calico.yaml b/templates/cluster-template-windows-calico.yaml index 690e13845..a036fd61f 100644 --- a/templates/cluster-template-windows-calico.yaml +++ b/templates/cluster-template-windows-calico.yaml @@ -3,7 +3,7 @@ kind: Cluster metadata: labels: cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - cni: flannel + cni: calico csi-proxy: enabled windows: enabled name: "${CLUSTER_NAME}" @@ -58,7 +58,6 @@ spec: certSANs: [localhost, 127.0.0.1] extraArgs: cloud-provider: oci - feature-gates: WindowsHostProcessContainers=true dns: {} etcd: {} networking: {} diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 37d376c02..afcd8a0a0 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -199,6 +199,29 @@ var _ = Describe("Workload cluster creation", func() { validateOLImage(namespace.Name, clusterName) }) + It("Windows - With 1 Linux control-plane nodes and with 1 Windows worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "windows-calico") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "windows-calico", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-windows-worker-nodes"), + }, result) + validateWindowsImage(namespace.Name, clusterName) + }) + It("Cloud Provider OCI testing [PRBlocking]", func() { clusterName = getClusterName(clusterNamePrefix, "ccm-testing") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -960,6 +983,33 @@ func validateOLImage(nameSpace string, clusterName string) { } } +func validateWindowsImage(nameSpace string, clusterName string) { + lister := bootstrapClusterProxy.GetClient() + inClustersNamespaceListOption := client.InNamespace(nameSpace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterLabelName: clusterName, + } + + machineList := &clusterv1.MachineList{} + Expect(lister.List(context.Background(), machineList, inClustersNamespaceListOption, matchClusterListOption)). + To(Succeed(), "Couldn't list machines for the cluster %q", clusterName) + + Expect(len(machineList.Items)).To(Equal(2)) + for _, machine := range machineList.Items { + if machine.Labels["os"] == "windows" { + instanceOcid := strings.Split(*machine.Spec.ProviderID, "//")[1] + Log(fmt.Sprintf("Instance OCID is %s", instanceOcid)) + resp, err := computeClient.GetInstance(context.Background(), core.GetInstanceRequest{ + InstanceId: common.String(instanceOcid), + }) + Expect(err).NotTo(HaveOccurred()) + instanceSourceDetails, ok := resp.SourceDetails.(core.InstanceSourceViaImageDetails) + Expect(ok).To(BeTrue()) + Expect(*instanceSourceDetails.ImageId).To(Equal(os.Getenv("OCI_WINDOWS_IMAGE_ID"))) + } + } +} + func getClusterName(prefix, specName string) string { clusterName := os.Getenv("CLUSTER_NAME") if clusterName == "" { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 468203a11..252404d38 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -70,6 +70,7 @@ providers: - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-externally-managed-vcn.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-machine-pool.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-windows-calico.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/metadata.yaml" variables: @@ -106,6 +107,7 @@ intervals: default/wait-cluster: ["30m", "10s"] default/wait-control-plane: ["30m", "10s"] default/wait-worker-nodes: ["30m", "10s"] + default/wait-windows-worker-nodes: ["60m", "30s"] default/wait-cluster-bare-metal: [ "60m", "10s" ] default/wait-control-plane-bare-metal: [ "60m", "10s" ] default/wait-worker-nodes-bare-metal: [ "60m", "10s" ] diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml new file mode 100644 index 000000000..b0e2d8424 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml @@ -0,0 +1,7 @@ +bases: +- ../bases/cluster.yaml +- ../bases/md.yaml +- ../bases/crs.yaml +- ../bases/ccm.yaml +patchesStrategicMerge: + - ./md.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml new file mode 100644 index 000000000..9f291624a --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml @@ -0,0 +1,46 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + labels: + os: windows +spec: + template: + spec: + imageId: "${OCI_WINDOWS_IMAGE_ID}" + shape: "BM.Standard.E4.128" + shapeConfig: + ocpus: "128" + vnicAttachments: + - displayName: "CalicoNic" + nicIndex: 1 # second nic must be used for hyper-v + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=false} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + provider-id: oci://{{ ds.meta_data["instance_id"] }} + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1 + - powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + failureDomain: "2" \ No newline at end of file