Add support for etcd scaling (#7127)

aws · Jan 17, 2024 · 1d4f649 · 1d4f649
1 parent 78be998
commit 1d4f649
Show file tree

Hide file tree

Showing 7 changed files with 397 additions and 53 deletions.
diff --git a/pkg/api/v1alpha1/cluster_webhook.go b/pkg/api/v1alpha1/cluster_webhook.go
@@ -325,16 +325,15 @@ func validateImmutableFieldsCluster(new, old *Cluster) field.ErrorList {
 	if new.Spec.ExternalEtcdConfiguration != nil && old.Spec.ExternalEtcdConfiguration == nil {
 		allErrs = append(
 			allErrs,
-			field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from local to external etcd topology"),
+			field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from stacked to external etcd topology"),
 		)
 	}
-	if new.Spec.ExternalEtcdConfiguration != nil && old.Spec.ExternalEtcdConfiguration != nil {
-		if old.Spec.ExternalEtcdConfiguration.Count != new.Spec.ExternalEtcdConfiguration.Count {
-			allErrs = append(
-				allErrs,
-				field.Forbidden(specPath.Child("externalEtcdConfiguration.count"), fmt.Sprintf("field is immutable %v", new.Spec.ExternalEtcdConfiguration.Count)),
-			)
-		}
+
+	if new.Spec.ExternalEtcdConfiguration == nil && old.Spec.ExternalEtcdConfiguration != nil {
+		allErrs = append(
+			allErrs,
+			field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from external to stacked etcd topology"),
+		)
 	}
 
 	if !new.Spec.GitOpsRef.Equal(old.Spec.GitOpsRef) && !old.IsSelfManaged() {

diff --git a/pkg/api/v1alpha1/cluster_webhook_test.go b/pkg/api/v1alpha1/cluster_webhook_test.go
@@ -419,18 +419,38 @@ func TestClusterValidateUpdateDatacenterRefNilImmutable(t *testing.T) {
 	g.Expect(c.ValidateUpdate(cOld)).To(MatchError(ContainSubstring("spec.datacenterRef: Forbidden: field is immutable")))
 }
 
-func TestClusterValidateUpdateExternalEtcdReplicasImmutable(t *testing.T) {
-	cOld := baseCluster()
-	cOld.Spec.ExternalEtcdConfiguration = &v1alpha1.ExternalEtcdConfiguration{Count: 3}
-	cOld.Spec.ControlPlaneConfiguration = v1alpha1.ControlPlaneConfiguration{
+func TestClusterValidateUpdateExternalEtcdConfiguration(t *testing.T) {
+	externalEtcdCluster := baseCluster()
+	externalEtcdCluster.Spec.ExternalEtcdConfiguration = &v1alpha1.ExternalEtcdConfiguration{Count: 3}
+	externalEtcdCluster.Spec.ControlPlaneConfiguration = v1alpha1.ControlPlaneConfiguration{
 		Count:    3,
 		Endpoint: &v1alpha1.Endpoint{Host: "1.1.1.1/1"},
 	}
-	c := cOld.DeepCopy()
-	c.Spec.ExternalEtcdConfiguration.Count = 5
+	stackedEtcdCluster := externalEtcdCluster.DeepCopy()
+	stackedEtcdCluster.Spec.ExternalEtcdConfiguration = nil
+	cases := map[string]struct {
+		cOld      *v1alpha1.Cluster
+		cNew      *v1alpha1.Cluster
+		expectErr string
+	}{
+		"local to external etcd configuration": {
+			cOld:      stackedEtcdCluster,
+			cNew:      externalEtcdCluster,
+			expectErr: "Forbidden: cannot switch from stacked to external etcd topology",
+		},
+		"external to local etcd configuration": {
+			cOld:      externalEtcdCluster,
+			cNew:      stackedEtcdCluster,
+			expectErr: "Forbidden: cannot switch from external to stacked etcd topology",
+		},
+	}
 
-	g := NewWithT(t)
-	g.Expect(c.ValidateUpdate(cOld)).To(MatchError(ContainSubstring("spec.externalEtcdConfiguration.count: Forbidden: field is immutable")))
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			g := NewWithT(t)
+			g.Expect(tt.cNew.ValidateUpdate(tt.cOld)).To(MatchError(ContainSubstring(tt.expectErr)))
+		})
+	}
 }
 
 func TestClusterValidateUpdateDataCenterRefNameImmutable(t *testing.T) {

diff --git a/pkg/validations/upgradevalidations/immutable_fields.go b/pkg/validations/upgradevalidations/immutable_fields.go
@@ -72,11 +72,7 @@ func ValidateImmutableFields(ctx context.Context, k validations.KubectlClient, c
 
 	oldETCD := oSpec.ExternalEtcdConfiguration
 	newETCD := nSpec.ExternalEtcdConfiguration
-	if oldETCD != nil && newETCD != nil {
-		if oldETCD.Count != newETCD.Count {
-			return errors.New("spec.externalEtcdConfiguration.count is immutable")
-		}
-	} else if oldETCD != newETCD {
+	if oldETCD != nil && newETCD == nil || oldETCD == nil && newETCD != nil {
 		return errors.New("adding or removing external etcd during upgrade is not supported")
 	}
 

diff --git a/pkg/validations/upgradevalidations/preflightvalidations_test.go b/pkg/validations/upgradevalidations/preflightvalidations_test.go
@@ -534,20 +534,6 @@ func TestPreflightValidationsVsphere(t *testing.T) {
 			crdResponse:        errors.New("error getting clusters crd: crd not found"),
 			wantErr:            explodingClusterError,
 		},
-		{
-			name:               "ValidationEtcdImmutable",
-			clusterVersion:     "v1.19.16-eks-1-19-4",
-			upgradeVersion:     "1.19",
-			getClusterResponse: goodClusterResponse,
-			cpResponse:         nil,
-			workerResponse:     nil,
-			nodeResponse:       nil,
-			crdResponse:        nil,
-			wantErr:            composeError("spec.externalEtcdConfiguration.count is immutable"),
-			modifyExistingSpecFunc: func(s *cluster.Spec) {
-				s.Cluster.Spec.ExternalEtcdConfiguration.Count++
-			},
-		},
 		{
 			name:               "ValidationControlPlaneImmutable",
 			clusterVersion:     "v1.19.16-eks-1-19-4",
@@ -903,24 +889,6 @@ func TestPreflightValidationsVsphere(t *testing.T) {
 				}
 			},
 		},
-		{
-			name:               "ValidationEtcdConfigReplicasImmutable",
-			clusterVersion:     "v1.19.16-eks-1-19-4",
-			upgradeVersion:     "1.19",
-			getClusterResponse: goodClusterResponse,
-			cpResponse:         nil,
-			workerResponse:     nil,
-			nodeResponse:       nil,
-			crdResponse:        nil,
-			wantErr:            composeError("spec.externalEtcdConfiguration.count is immutable"),
-			modifyExistingSpecFunc: func(s *cluster.Spec) {
-				s.Cluster.Spec.ExternalEtcdConfiguration.Count += 1
-				s.Cluster.Spec.DatacenterRef = anywherev1.Ref{
-					Kind: anywherev1.VSphereDatacenterKind,
-					Name: "vsphere test",
-				}
-			},
-		},
 		{
 			name:               "ValidationEtcdConfigPreviousSpecEmpty",
 			clusterVersion:     "v1.19.16-eks-1-19-4",

diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go
@@ -3447,3 +3447,94 @@ func TestCloudStackKubernetes128ValidateDomainFourLevelsSimpleFlow(t *testing.T)
 	)
 	runSimpleFlow(test)
 }
+
+// etcd scale tests
+func TestCloudstackKubernetes128EtcdScaleUp(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat128()),
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(1),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithExternalEtcdTopology(3),
+		),
+	)
+}
+
+func TestCloudstackKubernetes128EtcdScaleDown(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat128()),
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(3),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithExternalEtcdTopology(1),
+		),
+	)
+}
+
+func TestCloudstackKubernetes127to128EtcdScaleUp(t *testing.T) {
+	provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127())
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube127),
+			api.WithExternalEtcdTopology(1),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(3),
+		),
+		provider.WithProviderUpgrade(provider.Redhat128Template()),
+	)
+}
+
+func TestCloudstackKubernetes127to128EtcdScaleDown(t *testing.T) {
+	provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127())
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube127),
+			api.WithExternalEtcdTopology(3),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(1),
+		),
+		provider.WithProviderUpgrade(provider.Redhat128Template()),
+	)
+}
diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go
@@ -1212,3 +1212,92 @@ func TestDockerKubernetesUpgradeManagementComponents(t *testing.T) {
 	test.RunEKSA([]string{"upgrade", "management-components", "-f", test.ClusterConfigLocation, "-v", "99"})
 	test.DeleteCluster()
 }
+
+// etcd scale tests
+func TestDockerKubernetes128EtcdScaleUp(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewDocker(t),
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(1),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithExternalEtcdTopology(3),
+		),
+	)
+}
+
+func TestDockerKubernetes128EtcdScaleDown(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewDocker(t),
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(3),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithExternalEtcdTopology(1),
+		),
+	)
+}
+
+func TestDockerKubernetes127to128EtcdScaleUp(t *testing.T) {
+	provider := framework.NewDocker(t)
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube127),
+			api.WithExternalEtcdTopology(1),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(3),
+		),
+	)
+}
+
+func TestDockerKubernetes127to128EtcdScaleDown(t *testing.T) {
+	provider := framework.NewDocker(t)
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube127),
+			api.WithExternalEtcdTopology(3),
+			api.WithControlPlaneCount(1),
+			api.WithWorkerNodeCount(1),
+		),
+	)
+
+	runSimpleUpgradeFlow(
+		test,
+		v1alpha1.Kube128,
+		framework.WithClusterUpgrade(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(1),
+		),
+	)
+}