Add E2E tests for MachineDeployment orchestrated in-place upgrades (#74)

test/e2e/helpers.go

@@ -555,52 +555,58 @@ func WaitForControlPlaneAndMachinesReady(ctx context.Context, input WaitForContr
 }
 
 type ApplyInPlaceUpgradeAndWaitInput struct {
-	Getter                  framework.Getter
-	Machine                 *clusterv1.Machine
+	Getter framework.Getter
+	Obj    client.Object
+	// DestinationObj is used as a destination to decode whatever is retrieved from the client.
+	// e.g:
+	// 	{DestinationObj: &clusterv1.Machine{}, ...}
+	// 	client.Get(ctx, objKey, DestinationObj)
+	DestinationObj          client.Object
 	ClusterProxy            framework.ClusterProxy
 	UpgradeOption           string
 	WaitForUpgradeIntervals []interface{}
 }
 
+// ApplyInPlaceUpgradeAndWait applies an in-place upgrade to an object and waits for the upgrade to complete.
 func ApplyInPlaceUpgradeAndWait(ctx context.Context, input ApplyInPlaceUpgradeAndWaitInput) {
 	Expect(ctx).NotTo(BeNil())
-	Expect(input.Machine).ToNot(BeNil())
+	Expect(input.Obj).ToNot(BeNil())
+	Expect(input.DestinationObj).ToNot(BeNil())
 	Expect(input.ClusterProxy).ToNot(BeNil())
 	Expect(input.UpgradeOption).ToNot(BeEmpty())
 
 	mgmtClient := input.ClusterProxy.GetClient()
 
-	patchHelper, err := patch.NewHelper(input.Machine, mgmtClient)
+	patchHelper, err := patch.NewHelper(input.Obj, mgmtClient)
 	Expect(err).ToNot(HaveOccurred())
-	mAnnotations := input.Machine.GetAnnotations()
+	annotations := input.Obj.GetAnnotations()
 
-	if mAnnotations == nil {
-		mAnnotations = map[string]string{}
+	if annotations == nil {
+		annotations = map[string]string{}
 	}
 
-	mAnnotations[bootstrapv1.InPlaceUpgradeToAnnotation] = input.UpgradeOption
-	input.Machine.SetAnnotations(mAnnotations)
-	err = patchHelper.Patch(ctx, input.Machine)
+	annotations[bootstrapv1.InPlaceUpgradeToAnnotation] = input.UpgradeOption
+	input.Obj.SetAnnotations(annotations)
+	err = patchHelper.Patch(ctx, input.Obj)
 	Expect(err).ToNot(HaveOccurred())
 
 	By("Checking for in-place upgrade status to be equal to done")
 
 	Eventually(func() (bool, error) {
-		um := &clusterv1.Machine{}
-		if err := input.Getter.Get(ctx, client.ObjectKey{Namespace: input.Machine.Namespace, Name: input.Machine.Name}, um); err != nil {
+		if err := input.Getter.Get(ctx, client.ObjectKeyFromObject(input.Obj), input.DestinationObj); err != nil {
 			Byf("Failed to get the machine: %+v", err)
 			return false, err
 		}
 
-		mAnnotations := um.GetAnnotations()
+		mAnnotations := input.DestinationObj.GetAnnotations()
 
 		status, ok := mAnnotations[bootstrapv1.InPlaceUpgradeStatusAnnotation]
 		if !ok {
 			return false, nil
 		}
 
 		return status == bootstrapv1.InPlaceUpgradeDoneStatus, nil
-	}, input.WaitForUpgradeIntervals...).Should(BeTrue(), "In-place upgrade failed for %s", input.Machine.Name)
+	}, input.WaitForUpgradeIntervals...).Should(BeTrue(), "In-place upgrade failed for %s", input.Obj.GetName())
 }
 
 type ApplyInPlaceUpgradeForControlPlaneInput struct {
@@ -633,7 +639,8 @@ func ApplyInPlaceUpgradeForControlPlane(ctx context.Context, input ApplyInPlaceU
 	for _, machine := range machineList.Items {
 		ApplyInPlaceUpgradeAndWait(ctx, ApplyInPlaceUpgradeAndWaitInput{
 			Getter:                  input.Getter,
-			Machine:                 &machine,
+			Obj:                     &machine,
+			DestinationObj:          &clusterv1.Machine{},
 			ClusterProxy:            input.ClusterProxy,
 			UpgradeOption:           input.UpgradeOption,
 			WaitForUpgradeIntervals: input.WaitForUpgradeIntervals,
@@ -659,7 +666,7 @@ func ApplyInPlaceUpgradeForWorker(ctx context.Context, input ApplyInPlaceUpgrade
 	Expect(input.UpgradeOption).ToNot(BeEmpty())
 
 	for _, md := range input.MachineDeployments {
-		// Look up all the control plane machines.
+		// Look up all the worker machines.
 		inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace)
 		matchClusterListOption := client.MatchingLabels{
 			clusterv1.ClusterNameLabel:           input.Cluster.Name,
@@ -669,12 +676,13 @@ func ApplyInPlaceUpgradeForWorker(ctx context.Context, input ApplyInPlaceUpgrade
 		machineList := &clusterv1.MachineList{}
 		Eventually(func() error {
 			return input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)
-		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list control-plane machines for the cluster %q", input.Cluster.Name)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list worker machines for the cluster %q", input.Cluster.Name)
 
 		for _, machine := range machineList.Items {
 			ApplyInPlaceUpgradeAndWait(ctx, ApplyInPlaceUpgradeAndWaitInput{
 				Getter:                  input.Getter,
-				Machine:                 &machine,
+				Obj:                     &machine,
+				DestinationObj:          &clusterv1.Machine{},
 				ClusterProxy:            input.ClusterProxy,
 				UpgradeOption:           input.UpgradeOption,
 				WaitForUpgradeIntervals: input.WaitForUpgradeIntervals,
@@ -683,6 +691,59 @@ func ApplyInPlaceUpgradeForWorker(ctx context.Context, input ApplyInPlaceUpgrade
 	}
 }
 
+type ApplyInPlaceUpgradeForMachineDeploymentInput struct {
+	Lister                  framework.Lister
+	Getter                  framework.Getter
+	ClusterProxy            framework.ClusterProxy
+	Cluster                 *clusterv1.Cluster
+	MachineDeployments      []*clusterv1.MachineDeployment
+	UpgradeOption           string
+	WaitForUpgradeIntervals []interface{}
+}
+
+func ApplyInPlaceUpgradeForMachineDeployment(ctx context.Context, input ApplyInPlaceUpgradeForMachineDeploymentInput) {
+	Expect(ctx).NotTo(BeNil())
+	Expect(input.ClusterProxy).ToNot(BeNil())
+	Expect(input.Cluster).ToNot(BeNil())
+	Expect(input.MachineDeployments).ToNot(BeNil())
+	Expect(input.UpgradeOption).ToNot(BeEmpty())
+
+	var machineDeployment *clusterv1.MachineDeployment
+	for _, md := range input.MachineDeployments {
+		if md.Labels[clusterv1.ClusterNameLabel] == input.Cluster.Name {
+			machineDeployment = md
+			break
+		}
+	}
+	Expect(machineDeployment).ToNot(BeNil())
+
+	ApplyInPlaceUpgradeAndWait(ctx, ApplyInPlaceUpgradeAndWaitInput{
+		Getter:                  input.Getter,
+		Obj:                     machineDeployment,
+		DestinationObj:          &clusterv1.MachineDeployment{},
+		ClusterProxy:            input.ClusterProxy,
+		UpgradeOption:           input.UpgradeOption,
+		WaitForUpgradeIntervals: input.WaitForUpgradeIntervals,
+	})
+
+	// Make sure all the machines are upgraded
+	inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace)
+	matchClusterListOption := client.MatchingLabels{
+		clusterv1.ClusterNameLabel:           input.Cluster.Name,
+		clusterv1.MachineDeploymentNameLabel: machineDeployment.Name,
+	}
+
+	machineList := &clusterv1.MachineList{}
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list machines for the machineDeployment %q", machineDeployment.Name)
+
+	for _, machine := range machineList.Items {
+		Expect(machine.Annotations[bootstrapv1.InPlaceUpgradeStatusAnnotation]).To(Equal(bootstrapv1.InPlaceUpgradeDoneStatus))
+		Expect(machine.Annotations[bootstrapv1.InPlaceUpgradeReleaseAnnotation]).To(Equal(input.UpgradeOption))
+	}
+}
+
 // UpgradeControlPlaneAndWaitForUpgradeInput is the input type for UpgradeControlPlaneAndWaitForUpgrade.
 type UpgradeControlPlaneAndWaitForUpgradeInput struct {
 	ClusterProxy                framework.ClusterProxy

test/e2e/machine_deployment_test.go

@@ -0,0 +1,112 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+var _ = Describe("Machine Deployment Orchestrated In place upgrades", func() {
+	var (
+		ctx                    = context.TODO()
+		specName               = "workload-cluster-md-inplace"
+		namespace              *corev1.Namespace
+		cancelWatches          context.CancelFunc
+		result                 *ApplyClusterTemplateAndWaitResult
+		clusterName            string
+		clusterctlLogFolder    string
+		infrastructureProvider string
+	)
+
+	BeforeEach(func() {
+		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
+
+		clusterName = fmt.Sprintf("capick8s-md-in-place-%s", util.RandomString(6))
+		infrastructureProvider = clusterctl.DefaultInfrastructureProvider
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
+
+		result = new(ApplyClusterTemplateAndWaitResult)
+
+		clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName())
+	})
+
+	AfterEach(func() {
+		cleanInput := cleanupInput{
+			SpecName:        specName,
+			Cluster:         result.Cluster,
+			ClusterProxy:    bootstrapClusterProxy,
+			Namespace:       namespace,
+			CancelWatches:   cancelWatches,
+			IntervalsGetter: e2eConfig.GetIntervals,
+			SkipCleanup:     skipCleanup,
+			ArtifactFolder:  artifactFolder,
+		}
+
+		dumpSpecResourcesAndCleanup(ctx, cleanInput)
+	})
+
+	Context("Performing Machine Deployment Orchestrated in-place upgrades", func() {
+		It("Creating a workload cluster and applying in-place upgrade to Machine Deployment [MD-InPlace] [PR-Blocking]", func() {
+			By("Creating a workload cluster of 1 control plane and 3 worker nodes")
+			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
+				ClusterProxy: bootstrapClusterProxy,
+				ConfigCluster: clusterctl.ConfigClusterInput{
+					LogFolder:                clusterctlLogFolder,
+					ClusterctlConfigPath:     clusterctlConfigPath,
+					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
+					InfrastructureProvider:   infrastructureProvider,
+					Namespace:                namespace.Name,
+					ClusterName:              clusterName,
+					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+					ControlPlaneMachineCount: ptr.To(int64(1)),
+					WorkerMachineCount:       ptr.To(int64(3)),
+				},
+				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
+				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
+				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
+			}, result)
+
+			bootstrapProxyClient := bootstrapClusterProxy.GetClient()
+
+			By("Applying in place upgrade with local path for worker nodes")
+			ApplyInPlaceUpgradeForMachineDeployment(ctx, ApplyInPlaceUpgradeForMachineDeploymentInput{
+				Lister:                  bootstrapProxyClient,
+				Getter:                  bootstrapProxyClient,
+				ClusterProxy:            bootstrapClusterProxy,
+				Cluster:                 result.Cluster,
+				WaitForUpgradeIntervals: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"),
+				UpgradeOption:           e2eConfig.GetVariable(InPlaceUpgradeOption),
+				MachineDeployments:      result.MachineDeployments,
+			})
+		})
+	})
+
+})