Add a validation if the controller is paused during cluster upgrade

aws · Feb 14, 2024 · f17945d · f17945d
1 parent 01b8b83
commit f17945d
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 3 deletions.
diff --git a/pkg/validations/cluster.go b/pkg/validations/cluster.go
@@ -243,3 +243,15 @@ func ValidateEksaReleaseExistOnManagement(ctx context.Context, k kubernetes.Clie
 	}
 	return nil
 }
+
+// ValidatePauseAnnotation checks if the target cluster has annotation anywhere.eks.amazonaws.com/paused set to true or not.
+func ValidatePauseAnnotation(ctx context.Context, k KubectlClient, cluster *types.Cluster, clusterName string) error {
+	currentCluster, err := k.GetEksaCluster(ctx, cluster, clusterName)
+	if err != nil {
+		return err
+	}
+	if currentCluster.IsReconcilePaused() {
+		return fmt.Errorf("remove cluster controller reconciliation pause annotation %s before upgrading the cluster %s", currentCluster.PausedAnnotation(), clusterName)
+	}
+	return nil
+}
diff --git a/pkg/validations/cluster_test.go b/pkg/validations/cluster_test.go
@@ -618,3 +618,58 @@ func TestValidateEksaReleaseExistOnManagement(t *testing.T) {
 		})
 	}
 }
+
+func TestValidatePauseAnnotation(t *testing.T) {
+	mgmtName := "test"
+	tests := []struct {
+		name       string
+		gotCluster *anywherev1.Cluster
+		wantErr    error
+	}{
+		{
+			name: "success",
+			gotCluster: &anywherev1.Cluster{
+				ObjectMeta: v1.ObjectMeta{
+					Name:        mgmtName,
+					Annotations: map[string]string{},
+				},
+				Spec: anywherev1.ClusterSpec{
+					ManagementCluster: anywherev1.ManagementCluster{
+						Name: mgmtName,
+					},
+				},
+			},
+			wantErr: nil,
+		},
+		{
+			name: "success",
+			gotCluster: &anywherev1.Cluster{
+				ObjectMeta: v1.ObjectMeta{
+					Name:        mgmtName,
+					Annotations: map[string]string{"anywhere.eks.amazonaws.com/paused": "true"},
+				},
+				Spec: anywherev1.ClusterSpec{
+					ManagementCluster: anywherev1.ManagementCluster{
+						Name: mgmtName,
+					},
+				},
+			},
+			wantErr: fmt.Errorf("remove cluster controller reconciliation pause annotation %s before upgrading the cluster %s", "anywhere.eks.amazonaws.com/paused", mgmtName),
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tt := newTest(t, withKubectl())
+			ctx := context.Background()
+
+			tt.kubectl.EXPECT().GetEksaCluster(ctx, managementCluster(mgmtName), mgmtName).Return(tc.gotCluster, nil)
+
+			err := validations.ValidatePauseAnnotation(ctx, tt.kubectl, managementCluster(mgmtName), mgmtName)
+			if err != nil {
+				tt.Expect(err.Error()).To(ContainSubstring(tc.wantErr.Error()))
+			} else {
+				tt.Expect(tc.wantErr).To(BeNil())
+			}
+		})
+	}
+}
diff --git a/pkg/validations/upgradevalidations/preflightvalidations.go b/pkg/validations/upgradevalidations/preflightvalidations.go
@@ -24,6 +24,7 @@ func (u *UpgradeValidations) PreflightValidations(ctx context.Context) []validat
 		Name:           u.Opts.WorkloadCluster.Name,
 		KubeconfigFile: u.Opts.ManagementCluster.KubeconfigFile,
 	}
+
 	upgradeValidations := []validations.Validation{
 		func() *validations.ValidationResult {
 			return resultForRemediableValidation(
@@ -123,6 +124,13 @@ func (u *UpgradeValidations) PreflightValidations(ctx context.Context) []validat
 				Silent:      true,
 			}
 		},
+		func() *validations.ValidationResult {
+			return &validations.ValidationResult{
+				Name:        "validate eksa controller is not paused",
+				Remediation: "ensure cluster controller reconciliation is not paused manually before upgrading the cluster",
+				Err:         validations.ValidatePauseAnnotation(ctx, k, targetCluster, targetCluster.Name),
+			}
+		},
 	}
 
 	if u.Opts.Spec.Cluster.IsManaged() {

diff --git a/pkg/validations/upgradevalidations/preflightvalidations_test.go b/pkg/validations/upgradevalidations/preflightvalidations_test.go
@@ -389,7 +389,7 @@ func TestPreflightValidationsTinkerbell(t *testing.T) {
 			k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse)
 			k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse)
 			k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil)
-			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4)
+			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5)
 			upgradeValidations := upgradevalidations.New(opts)
 			err := validations.ProcessValidationResults(upgradeValidations.PreflightValidations(ctx))
 			if err != nil && err.Error() != tc.wantErr.Error() {
@@ -1116,7 +1116,7 @@ func TestPreflightValidationsVsphere(t *testing.T) {
 			k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse)
 			k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse)
 			k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil)
-			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4)
+			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5)
 			if opts.Spec.Cluster.IsManaged() {
 				k.EXPECT().GetEksaCluster(ctx, workloadCluster, workloadCluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4)
 			}
@@ -1354,7 +1354,7 @@ func TestPreFlightValidationsGit(t *testing.T) {
 			k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse)
 			k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse)
 			k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil)
-			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4)
+			k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5)
 			k.EXPECT().GetEksaFluxConfig(ctx, clusterSpec.Cluster.Spec.GitOpsRef.Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.FluxConfig, nil).MaxTimes(1)
 			k.EXPECT().GetEksaOIDCConfig(ctx, clusterSpec.Cluster.Spec.IdentityProviderRefs[0].Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.OIDCConfig, nil).MaxTimes(1)
 			k.EXPECT().GetEksaAWSIamConfig(ctx, clusterSpec.Cluster.Spec.IdentityProviderRefs[1].Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.AWSIamConfig, nil).MaxTimes(1)