From f17945d872559d7064ebb82ba02c4d6155971b22 Mon Sep 17 00:00:00 2001 From: Pankti Shah Date: Wed, 14 Feb 2024 13:34:31 -0800 Subject: [PATCH] Add a validation if the controller is paused during cluster upgrade --- pkg/validations/cluster.go | 12 ++++ pkg/validations/cluster_test.go | 55 +++++++++++++++++++ .../preflightvalidations.go | 8 +++ .../preflightvalidations_test.go | 6 +- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/pkg/validations/cluster.go b/pkg/validations/cluster.go index 8f0ab1ad3a2df..bba8459e8fb51 100644 --- a/pkg/validations/cluster.go +++ b/pkg/validations/cluster.go @@ -243,3 +243,15 @@ func ValidateEksaReleaseExistOnManagement(ctx context.Context, k kubernetes.Clie } return nil } + +// ValidatePauseAnnotation checks if the target cluster has annotation anywhere.eks.amazonaws.com/paused set to true or not. +func ValidatePauseAnnotation(ctx context.Context, k KubectlClient, cluster *types.Cluster, clusterName string) error { + currentCluster, err := k.GetEksaCluster(ctx, cluster, clusterName) + if err != nil { + return err + } + if currentCluster.IsReconcilePaused() { + return fmt.Errorf("remove cluster controller reconciliation pause annotation %s before upgrading the cluster %s", currentCluster.PausedAnnotation(), clusterName) + } + return nil +} diff --git a/pkg/validations/cluster_test.go b/pkg/validations/cluster_test.go index 2e2c3656dcb09..4e0f849ca1b9f 100644 --- a/pkg/validations/cluster_test.go +++ b/pkg/validations/cluster_test.go @@ -618,3 +618,58 @@ func TestValidateEksaReleaseExistOnManagement(t *testing.T) { }) } } + +func TestValidatePauseAnnotation(t *testing.T) { + mgmtName := "test" + tests := []struct { + name string + gotCluster *anywherev1.Cluster + wantErr error + }{ + { + name: "success", + gotCluster: &anywherev1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: mgmtName, + Annotations: map[string]string{}, + }, + Spec: anywherev1.ClusterSpec{ + ManagementCluster: anywherev1.ManagementCluster{ + Name: mgmtName, + }, + }, + }, + wantErr: nil, + }, + { + name: "success", + gotCluster: &anywherev1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: mgmtName, + Annotations: map[string]string{"anywhere.eks.amazonaws.com/paused": "true"}, + }, + Spec: anywherev1.ClusterSpec{ + ManagementCluster: anywherev1.ManagementCluster{ + Name: mgmtName, + }, + }, + }, + wantErr: fmt.Errorf("remove cluster controller reconciliation pause annotation %s before upgrading the cluster %s", "anywhere.eks.amazonaws.com/paused", mgmtName), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tt := newTest(t, withKubectl()) + ctx := context.Background() + + tt.kubectl.EXPECT().GetEksaCluster(ctx, managementCluster(mgmtName), mgmtName).Return(tc.gotCluster, nil) + + err := validations.ValidatePauseAnnotation(ctx, tt.kubectl, managementCluster(mgmtName), mgmtName) + if err != nil { + tt.Expect(err.Error()).To(ContainSubstring(tc.wantErr.Error())) + } else { + tt.Expect(tc.wantErr).To(BeNil()) + } + }) + } +} diff --git a/pkg/validations/upgradevalidations/preflightvalidations.go b/pkg/validations/upgradevalidations/preflightvalidations.go index 905de709db07e..ca261d8a30fec 100644 --- a/pkg/validations/upgradevalidations/preflightvalidations.go +++ b/pkg/validations/upgradevalidations/preflightvalidations.go @@ -24,6 +24,7 @@ func (u *UpgradeValidations) PreflightValidations(ctx context.Context) []validat Name: u.Opts.WorkloadCluster.Name, KubeconfigFile: u.Opts.ManagementCluster.KubeconfigFile, } + upgradeValidations := []validations.Validation{ func() *validations.ValidationResult { return resultForRemediableValidation( @@ -123,6 +124,13 @@ func (u *UpgradeValidations) PreflightValidations(ctx context.Context) []validat Silent: true, } }, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "validate eksa controller is not paused", + Remediation: "ensure cluster controller reconciliation is not paused manually before upgrading the cluster", + Err: validations.ValidatePauseAnnotation(ctx, k, targetCluster, targetCluster.Name), + } + }, } if u.Opts.Spec.Cluster.IsManaged() { diff --git a/pkg/validations/upgradevalidations/preflightvalidations_test.go b/pkg/validations/upgradevalidations/preflightvalidations_test.go index 8e0ac8882caca..5c8e9698a918b 100644 --- a/pkg/validations/upgradevalidations/preflightvalidations_test.go +++ b/pkg/validations/upgradevalidations/preflightvalidations_test.go @@ -389,7 +389,7 @@ func TestPreflightValidationsTinkerbell(t *testing.T) { k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse) k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse) k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil) - k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4) + k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5) upgradeValidations := upgradevalidations.New(opts) err := validations.ProcessValidationResults(upgradeValidations.PreflightValidations(ctx)) if err != nil && err.Error() != tc.wantErr.Error() { @@ -1116,7 +1116,7 @@ func TestPreflightValidationsVsphere(t *testing.T) { k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse) k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse) k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil) - k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4) + k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5) if opts.Spec.Cluster.IsManaged() { k.EXPECT().GetEksaCluster(ctx, workloadCluster, workloadCluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4) } @@ -1354,7 +1354,7 @@ func TestPreFlightValidationsGit(t *testing.T) { k.EXPECT().ValidateNodes(ctx, kubeconfigFilePath).Return(tc.nodeResponse) k.EXPECT().ValidateClustersCRD(ctx, workloadCluster).Return(tc.crdResponse) k.EXPECT().GetClusters(ctx, workloadCluster).Return(tc.getClusterResponse, nil) - k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(4) + k.EXPECT().GetEksaCluster(ctx, workloadCluster, clusterSpec.Cluster.Name).Return(existingClusterSpec.Cluster, nil).MaxTimes(5) k.EXPECT().GetEksaFluxConfig(ctx, clusterSpec.Cluster.Spec.GitOpsRef.Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.FluxConfig, nil).MaxTimes(1) k.EXPECT().GetEksaOIDCConfig(ctx, clusterSpec.Cluster.Spec.IdentityProviderRefs[0].Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.OIDCConfig, nil).MaxTimes(1) k.EXPECT().GetEksaAWSIamConfig(ctx, clusterSpec.Cluster.Spec.IdentityProviderRefs[1].Name, gomock.Any(), gomock.Any()).Return(existingClusterSpec.AWSIamConfig, nil).MaxTimes(1)