From 7d7b8744aca74e458d7c342f88c6a1f40113827f Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Mon, 23 Jan 2023 12:20:48 -0500 Subject: [PATCH] Add e2e test --- Makefile | 1 + api/v1beta1/conditions_consts.go | 2 + api/v1beta1/types.go | 4 +- cloud/scope/machine.go | 16 +- cloud/scope/util.go | 17 ++ cloud/scope/vnic_reconciler.go | 54 +---- cloud/scope/vnic_reconciler_test.go | 197 ++++++++++++++++++ .../compute/mock_compute/client_mock.go | 15 ++ cloud/services/vcn/mock_vcn/client_mock.go | 15 ++ ...tructure.cluster.x-k8s.io_ocimachines.yaml | 5 +- ....cluster.x-k8s.io_ocimachinetemplates.yaml | 6 +- controllers/ocimachine_controller.go | 5 +- .../cluster-template-windows-calico.yaml | 3 +- test/e2e/cluster_test.go | 50 +++++ test/e2e/config/e2e_conf.yaml | 2 + .../kustomization.yaml | 7 + .../cluster-template-windows-calico/md.yaml | 46 ++++ 17 files changed, 379 insertions(+), 66 deletions(-) create mode 100644 cloud/scope/vnic_reconciler_test.go create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml diff --git a/Makefile b/Makefile index 6b17a9a81..c10803662 100644 --- a/Makefile +++ b/Makefile @@ -252,6 +252,7 @@ generate-e2e-templates: $(KUSTOMIZE) $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-remote-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-remote-vcn-peering.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-externally-managed-vcn --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-externally-managed-vcn.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-machine-pool --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-machine-pool.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-windows-calico.yaml .PHONY: test-e2e-run test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests diff --git a/api/v1beta1/conditions_consts.go b/api/v1beta1/conditions_consts.go index c0abf8119..daf53b718 100644 --- a/api/v1beta1/conditions_consts.go +++ b/api/v1beta1/conditions_consts.go @@ -93,6 +93,8 @@ const ( RouteTableEventReady = "RouteTableReady" // SubnetEventReady used after reconciliation has completed successfully SubnetEventReady = "SubnetReady" + // InstanceVnicAttachmentReady used after reconciliation has been completed successfully + InstanceVnicAttachmentReady = "VnicAttachmentReady" // ApiServerLoadBalancerEventReady used after reconciliation has completed successfully ApiServerLoadBalancerEventReady = "APIServerLoadBalancerReady" // FailureDomainEventReady used after reconciliation has completed successfully diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 05427d592..663e9a50a 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -68,7 +68,7 @@ type VnicAttachment struct { // VnicAttachmentId defines the ID of the VnicAttachment VnicAttachmentId *string `json:"vnicAttachmentId,omitempty"` - // AssignPublicIp defines whether the instance should have a public IP address + // AssignPublicIp defines whether the vnic should have a public IP address // +optional AssignPublicIp bool `json:"assignPublicIp,omitempty"` @@ -77,7 +77,7 @@ type VnicAttachment struct { // +optional SubnetName string `json:"subnetName,omitempty"` - // DisplayName defines a user-friendly name. Does not have to be unique, and it's changeable. + // DisplayName defines a user-friendly name. Does not have to be unique. // Avoid entering confidential information. DisplayName *string `json:"displayName"` diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 75a2b5e0f..c5567cf63 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -220,16 +220,7 @@ func (m *MachineScope) GetOrCreateMachine(ctx context.Context) (*core.Instance, tags := m.getFreeFormTags(*m.OCICluster) - definedTags := make(map[string]map[string]interface{}) - if m.OCIMachine.Spec.DefinedTags != nil { - for ns, mapNs := range m.OCIMachine.Spec.DefinedTags { - mapValues := make(map[string]interface{}) - for k, v := range mapNs { - mapValues[k] = v - } - definedTags[ns] = mapValues - } - } + definedTags := ConvertMachineDefinedTags(m.OCIMachine.Spec.DefinedTags) availabilityDomain := m.OCICluster.Status.FailureDomains[*failureDomain].Attributes[AvailabilityDomain] faultDomain := m.OCICluster.Status.FailureDomains[*failureDomain].Attributes[FaultDomain] @@ -474,8 +465,10 @@ func (m *MachineScope) GetInstanceIp(ctx context.Context) (*string, error) { } } - if page = resp.OpcNextPage; resp.OpcNextPage == nil { + if resp.OpcNextPage == nil { break + } else { + page = resp.OpcNextPage } } @@ -624,7 +617,6 @@ func (m *MachineScope) getGetControlPlaneMachineNSGs() []string { // and returns the subnet ID if the name matches func (m *MachineScope) getMachineSubnet(name string) (*string, error) { for _, subnet := range m.OCICluster.Spec.NetworkSpec.Vcn.Subnets { - // if a subnet name is defined, use the correct subnet if subnet.Name == name { return subnet.ID, nil } diff --git a/cloud/scope/util.go b/cloud/scope/util.go index 0e49d5c73..92f7ee5f3 100644 --- a/cloud/scope/util.go +++ b/cloud/scope/util.go @@ -55,3 +55,20 @@ func GetSubnetNamesFromId(ids []string, subnets []*infrastructurev1beta1.Subnet) } return names } + +// ConvertMachineDefinedTags passes in the OCIMachineSpec DefinedTags and returns a converted map of defined tags +// to be used when creating API requests. +func ConvertMachineDefinedTags(machineDefinedTags map[string]map[string]string) map[string]map[string]interface{} { + definedTags := make(map[string]map[string]interface{}) + if machineDefinedTags != nil { + for ns, mapNs := range machineDefinedTags { + mapValues := make(map[string]interface{}) + for k, v := range mapNs { + mapValues[k] = v + } + definedTags[ns] = mapValues + } + } + + return definedTags +} diff --git a/cloud/scope/vnic_reconciler.go b/cloud/scope/vnic_reconciler.go index fb5b916d9..e8b700bb8 100644 --- a/cloud/scope/vnic_reconciler.go +++ b/cloud/scope/vnic_reconciler.go @@ -3,9 +3,9 @@ package scope import ( "context" "fmt" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" - "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/core" "github.com/pkg/errors" @@ -18,9 +18,7 @@ func (m *MachineScope) ReconcileVnicAttachments(ctx context.Context) error { for index, vnicAttachment := range m.OCIMachine.Spec.VnicAttachments { if m.vnicAttachmentExists(ctx, vnicAttachment) { - m.Logger.Info("vnicAttachment", vnicAttachment.DisplayName, " already exists") - // TODO: update vs create https://docs.oracle.com/en-us/iaas/api/#/en/iaas/20160918/Vnic/UpdateVnic - m.updateVnicAttachment(ctx, vnicAttachment) + m.Logger.Info("vnicAttachment", ociutil.DerefString(vnicAttachment.DisplayName), " already exists and is immutable") continue } @@ -31,9 +29,8 @@ func (m *MachineScope) ReconcileVnicAttachments(ctx context.Context) error { m.Logger.Error(err, msg) return err } - // TODO: kick off the "flush" so ids are stored as soon as they are attached + m.OCIMachine.Spec.VnicAttachments[index].VnicAttachmentId = vnicId - vnicAttachment.VnicAttachmentId = common.String("TESTING") } return nil @@ -54,16 +51,7 @@ func (m *MachineScope) createVnicAttachment(ctx context.Context, spec infrastruc tags := m.getFreeFormTags(*m.OCICluster) - definedTags := make(map[string]map[string]interface{}) - if m.OCIMachine.Spec.DefinedTags != nil { - for ns, mapNs := range m.OCIMachine.Spec.DefinedTags { - mapValues := make(map[string]interface{}) - for k, v := range mapNs { - mapValues[k] = v - } - definedTags[ns] = mapValues - } - } + definedTags := ConvertMachineDefinedTags(m.OCIMachine.Spec.DefinedTags) if spec.NicIndex == nil { spec.NicIndex = common.Int(0) @@ -85,38 +73,16 @@ func (m *MachineScope) createVnicAttachment(ctx context.Context, spec infrastruc }, } - retryToken := fmt.Sprintf("%s-%s", string(m.OCIMachine.UID), *vnicName) - req := core.AttachVnicRequest{AttachVnicDetails: secondVnic, - OpcRetryToken: ociutil.GetOPCRetryToken(retryToken)} + req := core.AttachVnicRequest{AttachVnicDetails: secondVnic} resp, err := m.ComputeClient.AttachVnic(ctx, req) - m.Logger.Info("AttachVnic resp: ", resp) - if err != nil { return nil, err } - return resp.VnicId, nil -} - -func (m *MachineScope) updateVnicAttachment(ctx context.Context, spec infrastructurev1beta1.VnicAttachment) error { - - req := core.UpdateVnicRequest{ - UpdateVnicDetails: core.UpdateVnicDetails{ - DisplayName: spec.DisplayName, - HostnameLabel: m.OCIMachine.Spec.NetworkDetails.HostnameLabel}, - VnicId: spec.VnicAttachmentId} - - // Send the request using the service client - resp, err := m.VCNClient.UpdateVnic(ctx, req) - if err != nil { - m.Logger.Error(err, "failed to reconcile the vcn, failed to update") - return errors.Wrap(err, "failed to reconcile the vcn, failed to update") - } - m.Logger.Info("successfully updated the vnicAttachment", "vcn", *resp.Id) - return nil + return resp.Id, nil } -func (m *MachineScope) vnicAttachmentExists(ctx context.Context, vnicName infrastructurev1beta1.VnicAttachment) bool { +func (m *MachineScope) vnicAttachmentExists(ctx context.Context, vnic infrastructurev1beta1.VnicAttachment) bool { found := false var page *string @@ -130,14 +96,16 @@ func (m *MachineScope) vnicAttachmentExists(ctx context.Context, vnicName infras return false } for _, attachment := range resp.Items { - if attachment.DisplayName == vnicName.DisplayName { + if ociutil.DerefString(attachment.DisplayName) == ociutil.DerefString(vnic.DisplayName) { m.Logger.Info("Vnic is already attached ", attachment) return true } } - if page = resp.OpcNextPage; resp.OpcNextPage == nil { + if resp.OpcNextPage == nil { break + } else { + page = resp.OpcNextPage } } return found diff --git a/cloud/scope/vnic_reconciler_test.go b/cloud/scope/vnic_reconciler_test.go new file mode 100644 index 000000000..f6449044d --- /dev/null +++ b/cloud/scope/vnic_reconciler_test.go @@ -0,0 +1,197 @@ +package scope + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/cluster-api-provider-oci/cloud/services/compute/mock_compute" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/core" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestReconcileVnicAttachment(t *testing.T) { + var ( + ms *MachineScope + mockCtrl *gomock.Controller + computeClient *mock_compute.MockComputeClient + ociCluster infrastructurev1beta1.OCICluster + ) + + setup := func(t *testing.T, g *WithT) { + var err error + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bootstrap", + Namespace: "default", + }, + Data: map[string][]byte{ + "value": []byte("test"), + }, + } + + mockCtrl = gomock.NewController(t) + computeClient = mock_compute.NewMockComputeClient(mockCtrl) + client := fake.NewClientBuilder().WithObjects(secret).Build() + ociCluster = infrastructurev1beta1.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "uid", + }, + Spec: infrastructurev1beta1.OCIClusterSpec{ + OCIResourceIdentifier: "resource_uid", + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ms, err = NewMachineScope(MachineScopeParams{ + ComputeClient: computeClient, + OCIMachine: &infrastructurev1beta1.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: infrastructurev1beta1.OCIMachineSpec{ + CompartmentId: "testCompartment", + VnicAttachments: []infrastructurev1beta1.VnicAttachment{ + { + DisplayName: common.String("VnicTest"), + NicIndex: common.Int(0), + }, + }, + }, + }, + Machine: &clusterv1.Machine{ + Spec: clusterv1.MachineSpec{ + Bootstrap: clusterv1.Bootstrap{ + DataSecretName: common.String("bootstrap"), + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + OCICluster: &ociCluster, + Client: client, + }) + ms.Machine.Namespace = "default" + g.Expect(err).To(BeNil()) + } + teardown := func(t *testing.T, g *WithT) { + mockCtrl.Finish() + + } + + tests := []struct { + name string + errorExpected bool + objects []client.Object + expectedEvent string + eventNotExpected string + matchError error + errorSubStringMatch bool + testSpecificSetup func(machineScope *MachineScope, computeClient *mock_compute.MockComputeClient) + }{ + { + name: "Crete vnic attachment", + errorExpected: false, + testSpecificSetup: func(machineScope *MachineScope, computeClient *mock_compute.MockComputeClient) { + ms.OCIMachine.Spec.InstanceId = common.String("test") + computeClient.EXPECT().ListVnicAttachments(gomock.Any(), gomock.Eq(core.ListVnicAttachmentsRequest{ + InstanceId: common.String("test"), + CompartmentId: common.String("testCompartment"), + })). + Return(core.ListVnicAttachmentsResponse{ + Items: []core.VnicAttachment{ + { + InstanceId: common.String("test"), + DisplayName: common.String("vnicDisplayName"), + }, + { + InstanceId: common.String("test"), + DisplayName: common.String("vnicDisplayName"), + }, + }, + }, nil) + + computeClient.EXPECT().AttachVnic(gomock.Any(), gomock.Eq(core.AttachVnicRequest{ + AttachVnicDetails: core.AttachVnicDetails{ + DisplayName: common.String("VnicTest"), + NicIndex: common.Int(0), + InstanceId: common.String("test"), + CreateVnicDetails: &core.CreateVnicDetails{ + DisplayName: common.String("VnicTest"), + AssignPublicIp: common.Bool(false), + DefinedTags: map[string]map[string]interface{}{}, + FreeformTags: map[string]string{ + ociutil.CreatedBy: ociutil.OCIClusterAPIProvider, + ociutil.ClusterResourceIdentifier: "resource_uid", + }, + NsgIds: nil, + }, + }})). + Return(core.AttachVnicResponse{ + VnicAttachment: core.VnicAttachment{Id: common.String("vnic.id")}, + }, nil) + }, + }, + { + name: "Crete vnic attachment error", + errorExpected: true, + matchError: fmt.Errorf("could not attach to nic 10"), + testSpecificSetup: func(machineScope *MachineScope, computeClient *mock_compute.MockComputeClient) { + ms.OCIMachine.Spec.InstanceId = common.String("test") + computeClient.EXPECT().ListVnicAttachments(gomock.Any(), gomock.Eq(core.ListVnicAttachmentsRequest{ + InstanceId: common.String("test"), + CompartmentId: common.String("testCompartment"), + })). + Return(core.ListVnicAttachmentsResponse{ + Items: []core.VnicAttachment{ + { + InstanceId: common.String("test"), + DisplayName: common.String("vnicDisplayName"), + NicIndex: common.Int(10), + }, + }, + }, nil) + + computeClient.EXPECT().AttachVnic(gomock.Any(), gomock.Any()). + Return(core.AttachVnicResponse{}, errors.New("could not attach to nic 10")) + }, + }, + { + name: "Crete vnic attachment on control plane will fail", + errorExpected: true, + matchError: fmt.Errorf("cannot attach multiple vnics to ControlPlane machines"), + testSpecificSetup: func(machineScope *MachineScope, computeClient *mock_compute.MockComputeClient) { + ms.Machine.ObjectMeta.Labels = make(map[string]string) + ms.Machine.ObjectMeta.Labels[clusterv1.MachineControlPlaneLabelName] = "Test" + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + defer teardown(t, g) + setup(t, g) + tc.testSpecificSetup(ms, computeClient) + err := ms.ReconcileVnicAttachments(context.Background()) + if tc.errorExpected { + g.Expect(err).To(Not(BeNil())) + if tc.errorSubStringMatch { + g.Expect(err.Error()).To(ContainSubstring(tc.matchError.Error())) + } else { + g.Expect(err.Error()).To(Equal(tc.matchError.Error())) + } + } else { + g.Expect(err).To(BeNil()) + } + }) + } +} diff --git a/cloud/services/compute/mock_compute/client_mock.go b/cloud/services/compute/mock_compute/client_mock.go index 9a3987d09..56ad89dfa 100644 --- a/cloud/services/compute/mock_compute/client_mock.go +++ b/cloud/services/compute/mock_compute/client_mock.go @@ -35,6 +35,21 @@ func (m *MockComputeClient) EXPECT() *MockComputeClientMockRecorder { return m.recorder } +// AttachVnic mocks base method. +func (m *MockComputeClient) AttachVnic(ctx context.Context, request core.AttachVnicRequest) (core.AttachVnicResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AttachVnic", ctx, request) + ret0, _ := ret[0].(core.AttachVnicResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AttachVnic indicates an expected call of AttachVnic. +func (mr *MockComputeClientMockRecorder) AttachVnic(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVnic", reflect.TypeOf((*MockComputeClient)(nil).AttachVnic), ctx, request) +} + // GetInstance mocks base method. func (m *MockComputeClient) GetInstance(ctx context.Context, request core.GetInstanceRequest) (core.GetInstanceResponse, error) { m.ctrl.T.Helper() diff --git a/cloud/services/vcn/mock_vcn/client_mock.go b/cloud/services/vcn/mock_vcn/client_mock.go index e80e7ff6c..eeb3fc219 100644 --- a/cloud/services/vcn/mock_vcn/client_mock.go +++ b/cloud/services/vcn/mock_vcn/client_mock.go @@ -575,6 +575,21 @@ func (mr *MockClientMockRecorder) GetVnic(ctx, request interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVnic", reflect.TypeOf((*MockClient)(nil).GetVnic), ctx, request) } +// UpdateVnic mocks base method. +func (m *MockClient) UpdateVnic(ctx context.Context, request core.UpdateVnicRequest) (core.UpdateVnicResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateVnic", ctx, request) + ret0, _ := ret[0].(core.UpdateVnicResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateVnic indicates an expected call of UpdateVnic. +func (mr *MockClientMockRecorder) UpdateVnic(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVnic", reflect.TypeOf((*MockClient)(nil).UpdateVnic), ctx, request) +} + // ListDrgAttachments mocks base method. func (m *MockClient) ListDrgAttachments(ctx context.Context, request core.ListDrgAttachmentsRequest) (core.ListDrgAttachmentsResponse, error) { m.ctrl.T.Helper() diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachines.yaml index 154231709..e78369106 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachines.yaml @@ -627,13 +627,12 @@ spec: items: properties: assignPublicIp: - description: AssignPublicIp defines whether the instance should + description: AssignPublicIp defines whether the vnic should have a public IP address type: boolean displayName: description: DisplayName defines a user-friendly name. Does - not have to be unique, and it's changeable. Avoid entering - confidential information. + not have to be unique. Avoid entering confidential information. type: string nicIndex: description: NicIndex defines which physical Network Interface diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachinetemplates.yaml index 63c85f9ae..fc48291f2 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimachinetemplates.yaml @@ -680,13 +680,13 @@ spec: items: properties: assignPublicIp: - description: AssignPublicIp defines whether the instance + description: AssignPublicIp defines whether the vnic should have a public IP address type: boolean displayName: description: DisplayName defines a user-friendly name. - Does not have to be unique, and it's changeable. Avoid - entering confidential information. + Does not have to be unique. Avoid entering confidential + information. type: string nicIndex: description: NicIndex defines which physical Network diff --git a/controllers/ocimachine_controller.go b/controllers/ocimachine_controller.go index 0a2b81e3c..d0bd6a830 100644 --- a/controllers/ocimachine_controller.go +++ b/controllers/ocimachine_controller.go @@ -294,10 +294,13 @@ func (r *OCIMachineReconciler) reconcileNormal(ctx context.Context, logger logr. err := machineScope.ReconcileVnicAttachments(ctx) if err != nil { r.Recorder.Event(machine, corev1.EventTypeWarning, "ReconcileError", errors.Wrapf(err, "failed to reconcile OCIMachine").Error()) - conditions.MarkFalse(machineScope.OCIMachine, infrastructurev1beta1.InstanceReadyCondition, infrastructurev1beta1.InstanceVnicAttachmentFailedReason, clusterv1.ConditionSeverityError, "") + conditions.MarkFalse(machineScope.OCIMachine, infrastructurev1beta1.InstanceReadyCondition, + infrastructurev1beta1.InstanceVnicAttachmentFailedReason, clusterv1.ConditionSeverityError, "") return ctrl.Result{}, err } machineScope.Info("Instance vnic attachment success") + r.Recorder.Eventf(machineScope.OCIMachine, corev1.EventTypeNormal, infrastructurev1beta1.InstanceVnicAttachmentReady, + "VNICs have been attached to instance.") } // record the event only when machine goes from not ready to ready state diff --git a/templates/cluster-template-windows-calico.yaml b/templates/cluster-template-windows-calico.yaml index 690e13845..a036fd61f 100644 --- a/templates/cluster-template-windows-calico.yaml +++ b/templates/cluster-template-windows-calico.yaml @@ -3,7 +3,7 @@ kind: Cluster metadata: labels: cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" - cni: flannel + cni: calico csi-proxy: enabled windows: enabled name: "${CLUSTER_NAME}" @@ -58,7 +58,6 @@ spec: certSANs: [localhost, 127.0.0.1] extraArgs: cloud-provider: oci - feature-gates: WindowsHostProcessContainers=true dns: {} etcd: {} networking: {} diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 37d376c02..1992f6fa8 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -199,6 +199,29 @@ var _ = Describe("Workload cluster creation", func() { validateOLImage(namespace.Name, clusterName) }) + It("Windows - With 1 Linux control-plane nodes and with 1 Windows worker nodes using Calico CNI", func() { + clusterName = getClusterName(clusterNamePrefix, "windows-calico") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "windows-calico", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-windows-worker-nodes"), + }, result) + validateWindowsImage(namespace.Name, clusterName) + }) + It("Cloud Provider OCI testing [PRBlocking]", func() { clusterName = getClusterName(clusterNamePrefix, "ccm-testing") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -960,6 +983,33 @@ func validateOLImage(nameSpace string, clusterName string) { } } +func validateWindowsImage(nameSpace string, clusterName string) { + lister := bootstrapClusterProxy.GetClient() + inClustersNamespaceListOption := client.InNamespace(nameSpace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterLabelName: clusterName, + } + + machineList := &clusterv1.MachineList{} + Expect(lister.List(context.Background(), machineList, inClustersNamespaceListOption, matchClusterListOption)). + To(Succeed(), "Couldn't list machines for the cluster %q", clusterName) + + Expect(len(machineList.Items)).To(Equal(2)) + for _, machine := range machineList.Items { + if machine.Labels["os"] == "windows" { + instanceOcid := strings.Split(*machine.Spec.ProviderID, "//")[1] + Log(fmt.Sprintf("Instance OCID is %s", instanceOcid)) + resp, err := computeClient.GetInstance(context.Background(), core.GetInstanceRequest{ + InstanceId: common.String(instanceOcid), + }) + Expect(err).NotTo(HaveOccurred()) + instanceSourceDetails, ok := resp.SourceDetails.(core.InstanceSourceViaImageDetails) + Expect(ok).To(BeTrue()) + Expect(*instanceSourceDetails.ImageId).To(Equal(os.Getenv("OCI_WINDOWS_IMAGE_ID"))) + } + } +} + func getClusterName(prefix, specName string) string { clusterName := os.Getenv("CLUSTER_NAME") if clusterName == "" { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 468203a11..252404d38 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -70,6 +70,7 @@ providers: - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-externally-managed-vcn.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-machine-pool.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-windows-calico.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/metadata.yaml" variables: @@ -106,6 +107,7 @@ intervals: default/wait-cluster: ["30m", "10s"] default/wait-control-plane: ["30m", "10s"] default/wait-worker-nodes: ["30m", "10s"] + default/wait-windows-worker-nodes: ["60m", "30s"] default/wait-cluster-bare-metal: [ "60m", "10s" ] default/wait-control-plane-bare-metal: [ "60m", "10s" ] default/wait-worker-nodes-bare-metal: [ "60m", "10s" ] diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml new file mode 100644 index 000000000..b0e2d8424 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/kustomization.yaml @@ -0,0 +1,7 @@ +bases: +- ../bases/cluster.yaml +- ../bases/md.yaml +- ../bases/crs.yaml +- ../bases/ccm.yaml +patchesStrategicMerge: + - ./md.yaml \ No newline at end of file diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml new file mode 100644 index 000000000..9f291624a --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-windows-calico/md.yaml @@ -0,0 +1,46 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + labels: + os: windows +spec: + template: + spec: + imageId: "${OCI_WINDOWS_IMAGE_ID}" + shape: "BM.Standard.E4.128" + shapeConfig: + ocpus: "128" + vnicAttachments: + - displayName: "CalicoNic" + nicIndex: 1 # second nic must be used for hyper-v + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=false} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + provider-id: oci://{{ ds.meta_data["instance_id"] }} + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - powershell C:\Windows\Setup\Scripts\enable_second_nic.ps1 + - powershell C:\Users\opc\attach_secondary_vnic.ps1 > C:\Users\opc\attach_secondary_vnic_log.txt +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + failureDomain: "2" \ No newline at end of file