From 7b80274392eb98bdea1851f76076e8d9110210e1 Mon Sep 17 00:00:00 2001 From: Xu Deng Date: Wed, 3 Jan 2024 10:42:23 -0500 Subject: [PATCH] Add upgrade management components cmd (#7238) --- .../cmd/upgrademanagementcomponents.go | 96 ++++++++ pkg/workflows/management/core_components.go | 26 ++- .../management/install_new_components.go | 15 +- .../upgrade_management_components.go | 206 ++++++++++++++++++ .../upgrade_management_components_test.go | 162 ++++++++++++++ 5 files changed, 492 insertions(+), 13 deletions(-) create mode 100644 cmd/eksctl-anywhere/cmd/upgrademanagementcomponents.go create mode 100644 pkg/workflows/management/upgrade_management_components.go create mode 100644 pkg/workflows/management/upgrade_management_components_test.go diff --git a/cmd/eksctl-anywhere/cmd/upgrademanagementcomponents.go b/cmd/eksctl-anywhere/cmd/upgrademanagementcomponents.go new file mode 100644 index 000000000000..7143a3a5879e --- /dev/null +++ b/cmd/eksctl-anywhere/cmd/upgrademanagementcomponents.go @@ -0,0 +1,96 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/aws/eks-anywhere/cmd/eksctl-anywhere/cmd/aflag" + "github.com/aws/eks-anywhere/pkg/dependencies" + "github.com/aws/eks-anywhere/pkg/kubeconfig" + "github.com/aws/eks-anywhere/pkg/types" + "github.com/aws/eks-anywhere/pkg/workflows/management" +) + +type upgradeManagementComponentsOptions struct { + clusterOptions +} + +var umco = &upgradeManagementComponentsOptions{} + +func init() { + flagSet := upgradeManagementComponentsCmd.Flags() + aflag.String(aflag.ClusterConfig, &umco.fileName, flagSet) + aflag.String(aflag.BundleOverride, &umco.bundlesOverride, flagSet) +} + +var upgradeManagementComponentsCmd = &cobra.Command{ + Use: "management-components", + Short: "Upgrade management components in a management cluster", + Long: "The term 'management components' encompasses all Kubernetes controllers and their CRDs present in the management cluster that are responsible for reconciling your EKS Anywhere (EKS-A) cluster. This command is specifically designed to facilitate the upgrade of these management components. Post this upgrade, the cluster itself can be upgraded by updating the 'eksaRelease' field in your eksa cluster object.", + PreRunE: bindFlagsToViper, + SilenceUsage: true, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + clusterSpec, err := newClusterSpec(clusterOptions{ + fileName: umco.fileName, + }) + if err != nil { + return err + } + if !clusterSpec.Cluster.IsSelfManaged() { + return fmt.Errorf("cluster %s doesn't contain management components to be upgraded", clusterSpec.Cluster.Name) + } + + cliConfig := buildCliConfig(clusterSpec) + dirs, err := uc.directoriesToMount(clusterSpec, cliConfig) + if err != nil { + return err + } + + factory := dependencies.ForSpec(clusterSpec).WithExecutableMountDirs(dirs...). + WithBootstrapper(). + WithCliConfig(cliConfig). + WithClusterManager(clusterSpec.Cluster, nil). + WithClusterApplier(). + WithProvider(umco.fileName, clusterSpec.Cluster, false, "", false, "", nil, nil). + WithGitOpsFlux(clusterSpec.Cluster, clusterSpec.FluxConfig, cliConfig). + WithWriter(). + WithCAPIManager(). + WithEksdUpgrader(). + WithEksdInstaller(). + WithKubectl(). + WithValidatorClients() + + deps, err := factory.Build(ctx) + if err != nil { + return err + } + defer close(cmd.Context(), deps) + + runner := management.NewUpgradeManagementComponentsRunner( + deps.Provider, + deps.CAPIManager, + deps.ClusterManager, + deps.GitOpsFlux, + deps.Writer, + deps.EksdUpgrader, + deps.EksdInstaller, + ) + + managementCluster := &types.Cluster{ + Name: clusterSpec.Cluster.Name, + KubeconfigFile: kubeconfig.FromClusterName(clusterSpec.Cluster.Name), + ExistingManagement: clusterSpec.Cluster.IsSelfManaged(), + } + + validator := management.NewUMCValidator(managementCluster, deps.Kubectl) + return runner.Run(ctx, clusterSpec, managementCluster, validator) + }, +} + +func init() { + upgradeCmd.AddCommand(upgradeManagementComponentsCmd) +} diff --git a/pkg/workflows/management/core_components.go b/pkg/workflows/management/core_components.go index a6efead267ae..e4a933a146cc 100644 --- a/pkg/workflows/management/core_components.go +++ b/pkg/workflows/management/core_components.go @@ -31,7 +31,7 @@ func (s *ensureEtcdCAPIComponentsExist) Checkpoint() *task.CompletedTask { } } -func (s *ensureEtcdCAPIComponentsExist) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) { +func (s *ensureEtcdCAPIComponentsExist) Restore(_ context.Context, _ *task.CommandContext, _ *task.CompletedTask) (task.Task, error) { return &pauseGitOpsReconcile{}, nil } @@ -39,8 +39,7 @@ type upgradeCoreComponents struct { UpgradeChangeDiff *types.ChangeDiff } -// Run upgradeCoreComponents upgrades pre cluster upgrade components. -func (s *upgradeCoreComponents) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { +func runUpgradeCoreComponents(ctx context.Context, commandContext *task.CommandContext) error { logger.Info("Upgrading core components") err := commandContext.Provider.PreCoreComponentsUpgrade( @@ -50,41 +49,50 @@ func (s *upgradeCoreComponents) Run(ctx context.Context, commandContext *task.Co ) if err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } changeDiff, err := commandContext.CAPIManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.Provider, commandContext.CurrentClusterSpec, commandContext.ClusterSpec) if err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } commandContext.UpgradeChangeDiff.Append(changeDiff) if err = commandContext.GitOpsManager.Install(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec); err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } changeDiff, err = commandContext.GitOpsManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec) if err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } commandContext.UpgradeChangeDiff.Append(changeDiff) changeDiff, err = commandContext.ClusterManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec) if err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } commandContext.UpgradeChangeDiff.Append(changeDiff) changeDiff, err = commandContext.EksdUpgrader.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec) if err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } commandContext.UpgradeChangeDiff.Append(changeDiff) + + return nil +} + +// Run upgradeCoreComponents upgrades pre cluster upgrade components. +func (s *upgradeCoreComponents) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { + if err := runUpgradeCoreComponents(ctx, commandContext); err != nil { + return &workflows.CollectMgmtClusterDiagnosticsTask{} + } s.UpgradeChangeDiff = commandContext.UpgradeChangeDiff return &preClusterUpgrade{} diff --git a/pkg/workflows/management/install_new_components.go b/pkg/workflows/management/install_new_components.go index de47e28fd01f..92c1c91b083c 100644 --- a/pkg/workflows/management/install_new_components.go +++ b/pkg/workflows/management/install_new_components.go @@ -9,21 +9,28 @@ import ( type installNewComponents struct{} -// Run installNewComponents performs actions needed to upgrade the management cluster. -func (s *installNewComponents) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { +func runInstallNewComponents(ctx context.Context, commandContext *task.CommandContext) error { if err := commandContext.ClusterManager.ApplyBundles(ctx, commandContext.ClusterSpec, commandContext.ManagementCluster); err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } if err := commandContext.ClusterManager.ApplyReleases(ctx, commandContext.ClusterSpec, commandContext.ManagementCluster); err != nil { commandContext.SetError(err) - return &workflows.CollectMgmtClusterDiagnosticsTask{} + return err } err := commandContext.EksdInstaller.InstallEksdManifest(ctx, commandContext.ClusterSpec, commandContext.ManagementCluster) if err != nil { commandContext.SetError(err) + return err + } + return nil +} + +// Run installNewComponents performs actions needed to upgrade the management cluster. +func (s *installNewComponents) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { + if err := runInstallNewComponents(ctx, commandContext); err != nil { return &workflows.CollectMgmtClusterDiagnosticsTask{} } return &upgradeCluster{} diff --git a/pkg/workflows/management/upgrade_management_components.go b/pkg/workflows/management/upgrade_management_components.go new file mode 100644 index 000000000000..083e3f2d0eda --- /dev/null +++ b/pkg/workflows/management/upgrade_management_components.go @@ -0,0 +1,206 @@ +package management + +import ( + "context" + "fmt" + + "github.com/aws/eks-anywhere/pkg/cluster" + "github.com/aws/eks-anywhere/pkg/executables" + "github.com/aws/eks-anywhere/pkg/filewriter" + "github.com/aws/eks-anywhere/pkg/logger" + "github.com/aws/eks-anywhere/pkg/providers" + "github.com/aws/eks-anywhere/pkg/task" + "github.com/aws/eks-anywhere/pkg/types" + "github.com/aws/eks-anywhere/pkg/validations" + "github.com/aws/eks-anywhere/pkg/workflows" + "github.com/aws/eks-anywhere/pkg/workflows/interfaces" +) + +// UpgradeManagementComponentsWorkflow is a schema for upgrade management components. +type UpgradeManagementComponentsWorkflow struct { + provider providers.Provider + clusterManager interfaces.ClusterManager + gitOpsManager interfaces.GitOpsManager + writer filewriter.FileWriter + capiManager interfaces.CAPIManager + eksdInstaller interfaces.EksdInstaller + eksdUpgrader interfaces.EksdUpgrader +} + +// NewUpgradeManagementComponentsRunner builds a new UpgradeManagementCommponents construct. +func NewUpgradeManagementComponentsRunner( + provider providers.Provider, + capiManager interfaces.CAPIManager, + clusterManager interfaces.ClusterManager, + gitOpsManager interfaces.GitOpsManager, + writer filewriter.FileWriter, + eksdUpgrader interfaces.EksdUpgrader, + eksdInstaller interfaces.EksdInstaller, +) *UpgradeManagementComponentsWorkflow { + return &UpgradeManagementComponentsWorkflow{ + provider: provider, + clusterManager: clusterManager, + gitOpsManager: gitOpsManager, + writer: writer, + capiManager: capiManager, + eksdUpgrader: eksdUpgrader, + eksdInstaller: eksdInstaller, + } +} + +// UMCValidator is a struct that holds a cluster and a kubectl executable. +// It is used to perform preflight validations on the cluster. +type UMCValidator struct { + cluster *types.Cluster + kubectl *executables.Kubectl +} + +// NewUMCValidator is a constructor function that creates a new instance of UMCValidator. +func NewUMCValidator(cluster *types.Cluster, kubectl *executables.Kubectl) *UMCValidator { + return &UMCValidator{ + cluster: cluster, + kubectl: kubectl, + } +} + +// PreflightValidations is a method of the UMCValidator struct. +// It performs preflight validations on the cluster and returns a slice of Validation objects. +func (u *UMCValidator) PreflightValidations(ctx context.Context) []validations.Validation { + return []validations.Validation{ + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "control plane ready", + Remediation: fmt.Sprintf("ensure control plane nodes and pods for cluster %s are Ready", u.cluster.Name), + Err: u.kubectl.ValidateControlPlaneNodes(ctx, u.cluster, u.cluster.Name), + } + }, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "cluster CRDs ready", + Remediation: "", + Err: u.kubectl.ValidateClustersCRD(ctx, u.cluster), + } + }, + } +} + +// Run Upgrade implements upgrade functionality for management cluster's upgrade operation. +func (umc *UpgradeManagementComponentsWorkflow) Run(ctx context.Context, clusterSpec *cluster.Spec, managementCluster *types.Cluster, validator interfaces.Validator) error { + commandContext := &task.CommandContext{ + Provider: umc.provider, + ClusterManager: umc.clusterManager, + ManagementCluster: managementCluster, + ClusterSpec: clusterSpec, + Validations: validator, + Writer: umc.writer, + CAPIManager: umc.capiManager, + UpgradeChangeDiff: types.NewChangeDiff(), + GitOpsManager: umc.gitOpsManager, + EksdUpgrader: umc.eksdUpgrader, + EksdInstaller: umc.eksdInstaller, + } + + return task.NewTaskRunner(&setupAndValidateMC{}, umc.writer).RunTask(ctx, commandContext) +} + +type setupAndValidateMC struct{} + +// Run setupAndValidate validates management cluster before upgrade process starts. +func (s *setupAndValidateMC) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { + logger.Info("Performing setup and validations") + currentSpec, err := commandContext.ClusterManager.GetCurrentClusterSpec(ctx, commandContext.ManagementCluster, commandContext.ClusterSpec.Cluster.Name) + if err != nil { + commandContext.SetError(err) + return nil + } + commandContext.CurrentClusterSpec = currentSpec + runner := validations.NewRunner() + runner.Register( + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: fmt.Sprintf("%s provider validation", commandContext.Provider.Name()), + Err: commandContext.Provider.SetupAndValidateUpgradeCluster(ctx, commandContext.ManagementCluster, commandContext.ClusterSpec, commandContext.CurrentClusterSpec), + } + }, + ) + runner.Register(commandContext.Validations.PreflightValidations(ctx)...) + + err = runner.Run() + if err != nil { + commandContext.SetError(err) + return nil + } + + return &upgradeCoreComponentsMC{ + UpgradeChangeDiff: &types.ChangeDiff{}, + } +} + +func (s *setupAndValidateMC) Name() string { + return "validate" +} + +func (s *setupAndValidateMC) Restore(_ context.Context, _ *task.CommandContext, _ *task.CompletedTask) (task.Task, error) { + return nil, nil +} + +func (s *setupAndValidateMC) Checkpoint() *task.CompletedTask { + return &task.CompletedTask{ + Checkpoint: nil, + } +} + +// This struct is similar to upgradeCoreComponents, but its returned value is different in Run() function. +type upgradeCoreComponentsMC struct { + UpgradeChangeDiff *types.ChangeDiff +} + +func (s *upgradeCoreComponentsMC) Name() string { + return "upgrade-core-components-mc" +} + +func (s *upgradeCoreComponentsMC) Checkpoint() *task.CompletedTask { + return &task.CompletedTask{ + Checkpoint: s.UpgradeChangeDiff, + } +} + +func (s *upgradeCoreComponentsMC) Restore(_ context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) { + s.UpgradeChangeDiff = &types.ChangeDiff{} + if err := task.UnmarshalTaskCheckpoint(completedTask.Checkpoint, s.UpgradeChangeDiff); err != nil { + return nil, err + } + commandContext.UpgradeChangeDiff = s.UpgradeChangeDiff + return &installNewComponentsMC{}, nil +} + +func (s *upgradeCoreComponentsMC) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { + if err := runUpgradeCoreComponents(ctx, commandContext); err != nil { + return &workflows.CollectMgmtClusterDiagnosticsTask{} + } + return &installNewComponentsMC{} +} + +// This struct is similar to installNewComponents, but its returned value is different in Run() function. +type installNewComponentsMC struct{} + +func (s *installNewComponentsMC) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { + if err := runInstallNewComponents(ctx, commandContext); err != nil { + return &workflows.CollectMgmtClusterDiagnosticsTask{} + } + return nil +} + +func (s *installNewComponentsMC) Name() string { + return "install-new-eksa-version-components-mc" +} + +func (s *installNewComponentsMC) Checkpoint() *task.CompletedTask { + return &task.CompletedTask{ + Checkpoint: nil, + } +} + +func (s *installNewComponentsMC) Restore(_ context.Context, _ *task.CommandContext, _ *task.CompletedTask) (task.Task, error) { + return nil, nil +} diff --git a/pkg/workflows/management/upgrade_management_components_test.go b/pkg/workflows/management/upgrade_management_components_test.go new file mode 100644 index 000000000000..7578fdb144c3 --- /dev/null +++ b/pkg/workflows/management/upgrade_management_components_test.go @@ -0,0 +1,162 @@ +package management + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + + "github.com/aws/eks-anywhere/internal/test" + writermocks "github.com/aws/eks-anywhere/pkg/filewriter/mocks" + "github.com/aws/eks-anywhere/pkg/kubeconfig" + providermocks "github.com/aws/eks-anywhere/pkg/providers/mocks" + "github.com/aws/eks-anywhere/pkg/types" + "github.com/aws/eks-anywhere/pkg/validations" + "github.com/aws/eks-anywhere/pkg/workflows/interfaces/mocks" +) + +var capiChangeDiff = types.NewChangeDiff(&types.ComponentChangeDiff{ + ComponentName: "vsphere", + OldVersion: "v0.0.1", + NewVersion: "v0.0.2", +}) + +var fluxChangeDiff = types.NewChangeDiff(&types.ComponentChangeDiff{ + ComponentName: "Flux", + OldVersion: "v0.0.1", + NewVersion: "v0.0.2", +}) + +var eksaChangeDiff = types.NewChangeDiff(&types.ComponentChangeDiff{ + ComponentName: "eks-a", + OldVersion: "v0.0.1", + NewVersion: "v0.0.2", +}) + +var eksdChangeDiff = types.NewChangeDiff(&types.ComponentChangeDiff{ + ComponentName: "eks-d", + OldVersion: "v0.0.1", + NewVersion: "v0.0.2", +}) + +type TestMocks struct { + mockCtrl *gomock.Controller + clusterManager *mocks.MockClusterManager + gitOpsManager *mocks.MockGitOpsManager + provider *providermocks.MockProvider + writer *writermocks.MockFileWriter + eksdInstaller *mocks.MockEksdInstaller + eksdUpgrader *mocks.MockEksdUpgrader + capiManager *mocks.MockCAPIManager + validator *mocks.MockValidator +} + +func NewTestMocks(t *testing.T) *TestMocks { + mockCtrl := gomock.NewController(t) + return &TestMocks{ + mockCtrl: mockCtrl, + clusterManager: mocks.NewMockClusterManager(mockCtrl), + gitOpsManager: mocks.NewMockGitOpsManager(mockCtrl), + provider: providermocks.NewMockProvider(mockCtrl), + writer: writermocks.NewMockFileWriter(mockCtrl), + eksdInstaller: mocks.NewMockEksdInstaller(mockCtrl), + eksdUpgrader: mocks.NewMockEksdUpgrader(mockCtrl), + capiManager: mocks.NewMockCAPIManager(mockCtrl), + validator: mocks.NewMockValidator(mockCtrl), + } +} + +func TestRunnerHappyPath(t *testing.T) { + mocks := NewTestMocks(t) + runner := NewUpgradeManagementComponentsRunner( + mocks.provider, + mocks.capiManager, + mocks.clusterManager, + mocks.gitOpsManager, + mocks.writer, + mocks.eksdUpgrader, + mocks.eksdInstaller, + ) + + clusterSpec := test.NewClusterSpec() + managementCluster := &types.Cluster{ + Name: clusterSpec.Cluster.Name, + KubeconfigFile: kubeconfig.FromClusterName(clusterSpec.Cluster.Name), + ExistingManagement: clusterSpec.Cluster.IsSelfManaged(), + } + + ctx := context.Background() + curSpec := test.NewClusterSpec() + newSpec := test.NewClusterSpec() + + mocks.clusterManager.EXPECT().GetCurrentClusterSpec(ctx, gomock.Any(), managementCluster.Name).Return(curSpec, nil) + gomock.InOrder( + mocks.validator.EXPECT().PreflightValidations(ctx).Return(nil), + mocks.provider.EXPECT().Name(), + mocks.provider.EXPECT().SetupAndValidateUpgradeCluster(ctx, gomock.Any(), newSpec, curSpec), + mocks.provider.EXPECT().PreCoreComponentsUpgrade(gomock.Any(), gomock.Any(), gomock.Any()), + mocks.capiManager.EXPECT().Upgrade(ctx, managementCluster, mocks.provider, curSpec, newSpec).Return(capiChangeDiff, nil), + mocks.gitOpsManager.EXPECT().Install(ctx, managementCluster, curSpec, newSpec).Return(nil), + mocks.gitOpsManager.EXPECT().Upgrade(ctx, managementCluster, curSpec, newSpec).Return(fluxChangeDiff, nil), + mocks.clusterManager.EXPECT().Upgrade(ctx, managementCluster, curSpec, newSpec).Return(eksaChangeDiff, nil), + mocks.eksdUpgrader.EXPECT().Upgrade(ctx, managementCluster, curSpec, newSpec).Return(eksdChangeDiff, nil), + mocks.clusterManager.EXPECT().ApplyBundles( + ctx, newSpec, managementCluster, + ).Return(nil), + mocks.clusterManager.EXPECT().ApplyReleases( + ctx, newSpec, managementCluster, + ).Return(nil), + mocks.eksdInstaller.EXPECT().InstallEksdManifest( + ctx, newSpec, managementCluster, + ).Return(nil), + ) + + err := runner.Run(ctx, newSpec, managementCluster, mocks.validator) + if err != nil { + t.Fatalf("UpgradeManagementComponents.Run() err = %v, want err = nil", err) + } +} + +func TestRunnerStopsWhenValidationFailed(t *testing.T) { + mocks := NewTestMocks(t) + runner := NewUpgradeManagementComponentsRunner( + mocks.provider, + mocks.capiManager, + mocks.clusterManager, + mocks.gitOpsManager, + mocks.writer, + mocks.eksdUpgrader, + mocks.eksdInstaller, + ) + + clusterSpec := test.NewClusterSpec() + managementCluster := &types.Cluster{ + Name: clusterSpec.Cluster.Name, + KubeconfigFile: kubeconfig.FromClusterName(clusterSpec.Cluster.Name), + ExistingManagement: clusterSpec.Cluster.IsSelfManaged(), + } + + ctx := context.Background() + curSpec := test.NewClusterSpec() + newSpec := test.NewClusterSpec() + + mocks.provider.EXPECT().Name() + mocks.provider.EXPECT().SetupAndValidateUpgradeCluster(ctx, gomock.Any(), newSpec, curSpec) + mocks.clusterManager.EXPECT().GetCurrentClusterSpec(ctx, gomock.Any(), managementCluster.Name).Return(curSpec, nil) + mocks.validator.EXPECT().PreflightValidations(ctx).Return( + []validations.Validation{ + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Err: errors.New("validation failed"), + } + }, + }) + + mocks.writer.EXPECT().Write(fmt.Sprintf("%s-checkpoint.yaml", newSpec.Cluster.Name), gomock.Any()) + err := runner.Run(ctx, newSpec, managementCluster, mocks.validator) + if err == nil { + t.Fatalf("UpgradeManagementComponents.Run() err == nil, want err != nil") + } +}