Skip to content

Commit

Permalink
Add delete mgmt workflow (#7502)
Browse files Browse the repository at this point in the history
  • Loading branch information
tatlat authored Feb 16, 2024
1 parent 595507f commit a590acd
Show file tree
Hide file tree
Showing 20 changed files with 913 additions and 90 deletions.
16 changes: 6 additions & 10 deletions cmd/eksctl-anywhere/cmd/deletecluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"github.com/aws/eks-anywhere/pkg/providers/tinkerbell/hardware"
"github.com/aws/eks-anywhere/pkg/types"
"github.com/aws/eks-anywhere/pkg/validations"
"github.com/aws/eks-anywhere/pkg/workflows"
"github.com/aws/eks-anywhere/pkg/workflows/management"
"github.com/aws/eks-anywhere/pkg/workflows/workload"
)

Expand Down Expand Up @@ -123,6 +123,9 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
WithWriter().
WithDeleteClusterDefaulter(deleteCLIConfig).
WithClusterDeleter().
WithEksdInstaller().
WithEKSAInstaller().
WithUnAuthKubeClient().
Build(ctx)
if err != nil {
return err
Expand All @@ -134,14 +137,6 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
return err
}

deleteCluster := workflows.NewDelete(
deps.Bootstrapper,
deps.Provider,
deps.ClusterManager,
deps.GitOpsFlux,
deps.Writer,
)

var cluster *types.Cluster
if clusterSpec.ManagementCluster == nil {
cluster = &types.Cluster{
Expand All @@ -159,7 +154,8 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
deleteWorkload := workload.NewDelete(deps.Provider, deps.Writer, deps.ClusterManager, deps.ClusterDeleter, deps.GitOpsFlux)
err = deleteWorkload.Run(ctx, cluster, clusterSpec)
} else {
err = deleteCluster.Run(ctx, cluster, clusterSpec, dc.forceCleanup, dc.managementKubeconfig)
deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient)
err = deleteManagement.Run(ctx, cluster, clusterSpec)
}
cleanup(deps, &err)
return err
Expand Down
9 changes: 1 addition & 8 deletions controllers/cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
kerrors "k8s.io/apimachinery/pkg/util/errors"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
Expand Down Expand Up @@ -464,16 +463,10 @@ func (r *ClusterReconciler) updateStatus(ctx context.Context, log logr.Logger, c
}

func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (ctrl.Result, error) {
if cluster.IsSelfManaged() {
if cluster.IsSelfManaged() && !cluster.IsManagedByCLI() {
return ctrl.Result{}, errors.New("deleting self-managed clusters is not supported")
}

if metav1.HasAnnotation(cluster.ObjectMeta, anywherev1.ManagedByCLIAnnotation) {
log.Info("Clusters is managed by CLI, removing finalizer")
controllerutil.RemoveFinalizer(cluster, ClusterFinalizerName)
return ctrl.Result{}, nil
}

if cluster.IsReconcilePaused() {
log.Info("Cluster reconciliation is paused, won't process cluster deletion")
return ctrl.Result{}, nil
Expand Down
40 changes: 0 additions & 40 deletions controllers/cluster_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1140,46 +1140,6 @@ func TestClusterReconcilerReconcileDeletePausedCluster(t *testing.T) {
})
}

func TestClusterReconcilerReconcileDeleteClusterManagedByCLI(t *testing.T) {
g := NewWithT(t)
ctx := context.Background()
managementCluster := vsphereCluster()
managementCluster.Name = "management-cluster"
cluster := vsphereCluster()
cluster.SetManagedBy(managementCluster.Name)
controllerutil.AddFinalizer(cluster, controllers.ClusterFinalizerName)
capiCluster := newCAPICluster(cluster.Name, cluster.Namespace)

// Mark cluster for deletion
now := metav1.Now()
cluster.DeletionTimestamp = &now

// Mark as managed by CLI
cluster.Annotations[anywherev1.ManagedByCLIAnnotation] = "true"

c := fake.NewClientBuilder().WithRuntimeObjects(
managementCluster, cluster, capiCluster,
).Build()
controller := gomock.NewController(t)
iam := mocks.NewMockAWSIamConfigReconciler(controller)
clusterValidator := mocks.NewMockClusterValidator(controller)
mhcReconciler := mocks.NewMockMachineHealthCheckReconciler(controller)

r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil, mhcReconciler)
g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
api := envtest.NewAPIExpecter(t, c)

cl := envtest.CloneNameNamespace(cluster)
api.ShouldEventuallyNotExist(ctx, cl)

capiCl := envtest.CloneNameNamespace(capiCluster)
api.ShouldEventuallyMatch(ctx, capiCl, func(g Gomega) {
g.Expect(
capiCluster.DeletionTimestamp.IsZero(),
).To(BeTrue(), "CAPI cluster should exist and not be marked for deletion")
})
}

func TestClusterReconcilerDeleteNoCAPIClusterSuccess(t *testing.T) {
g := NewWithT(t)

Expand Down
13 changes: 13 additions & 0 deletions pkg/clustermanager/cluster_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,19 @@ func (c *ClusterManager) RemoveManagedByCLIAnnotationForCluster(ctx context.Cont
return nil
}

// AddManagedByCLIAnnotationForCluster removes the managed-by-cli annotation from the cluster.
func (c *ClusterManager) AddManagedByCLIAnnotationForCluster(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
err := c.clusterClient.UpdateAnnotationInNamespace(ctx, clusterSpec.Cluster.ResourceType(),
cluster.Name,
map[string]string{v1alpha1.ManagedByCLIAnnotation: "true"},
cluster,
clusterSpec.Cluster.Namespace)
if err != nil {
return fmt.Errorf("adding managed by CLI annotation after apply cluster spec: %v", err)
}
return nil
}

func (c *ClusterManager) applyResource(ctx context.Context, cluster *types.Cluster, resourcesSpec []byte) error {
err := c.clusterClient.ApplyKubeSpecFromBytes(ctx, cluster, resourcesSpec)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions pkg/workflows/interfaces/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type ClusterManager interface {
GenerateAWSIAMKubeconfig(ctx context.Context, cluster *types.Cluster) error
DeletePackageResources(ctx context.Context, managementCluster *types.Cluster, clusterName string) error
CreateRegistryCredSecret(ctx context.Context, mgmt *types.Cluster) error
AddManagedByCLIAnnotationForCluster(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error
}

type GitOpsManager interface {
Expand Down
14 changes: 14 additions & 0 deletions pkg/workflows/interfaces/mocks/clients.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

68 changes: 68 additions & 0 deletions pkg/workflows/management/delete.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package management

import (
"context"

"github.com/aws/eks-anywhere/pkg/cluster"
"github.com/aws/eks-anywhere/pkg/filewriter"
"github.com/aws/eks-anywhere/pkg/providers"
"github.com/aws/eks-anywhere/pkg/task"
"github.com/aws/eks-anywhere/pkg/types"
"github.com/aws/eks-anywhere/pkg/workflows/interfaces"
)

// Delete is the workflow that deletes a workload clusters.
type Delete struct {
bootstrapper interfaces.Bootstrapper
provider providers.Provider
writer filewriter.FileWriter
clusterManager interfaces.ClusterManager
gitopsManager interfaces.GitOpsManager
clusterDeleter interfaces.ClusterDeleter
eksdInstaller interfaces.EksdInstaller
eksaInstaller interfaces.EksaInstaller
clientFactory interfaces.ClientFactory
}

// NewDelete builds a new delete construct.
func NewDelete(bootstrapper interfaces.Bootstrapper,
provider providers.Provider,
writer filewriter.FileWriter,
clusterManager interfaces.ClusterManager,
gitopsManager interfaces.GitOpsManager,
clusterDeleter interfaces.ClusterDeleter,
eksdInstaller interfaces.EksdInstaller,
eksaInstaller interfaces.EksaInstaller,
clientFactory interfaces.ClientFactory,
) *Delete {
return &Delete{
bootstrapper: bootstrapper,
provider: provider,
writer: writer,
clusterManager: clusterManager,
gitopsManager: gitopsManager,
clusterDeleter: clusterDeleter,
eksdInstaller: eksdInstaller,
eksaInstaller: eksaInstaller,
clientFactory: clientFactory,
}
}

// Run executes the tasks to delete a management cluster.
func (c *Delete) Run(ctx context.Context, workload *types.Cluster, clusterSpec *cluster.Spec) error {
commandContext := &task.CommandContext{
Bootstrapper: c.bootstrapper,
Provider: c.provider,
Writer: c.writer,
ClusterManager: c.clusterManager,
ClusterSpec: clusterSpec,
WorkloadCluster: workload,
GitOpsManager: c.gitopsManager,
ClusterDeleter: c.clusterDeleter,
EksdInstaller: c.eksdInstaller,
EksaInstaller: c.eksaInstaller,
ClientFactory: c.clientFactory,
}

return task.NewTaskRunner(&setupAndValidateDelete{}, c.writer).RunTask(ctx, commandContext)
}
38 changes: 38 additions & 0 deletions pkg/workflows/management/delete_bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package management

import (
"context"

"github.com/aws/eks-anywhere/pkg/constants"
"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/task"
"github.com/aws/eks-anywhere/pkg/workflows"
)

type deleteBootstrapClusterForDeleteTask struct{}

func (s *deleteBootstrapClusterForDeleteTask) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Deleting bootstrap cluster")
if err := commandContext.Bootstrapper.DeleteBootstrapCluster(ctx, commandContext.BootstrapCluster, constants.Delete, false); err != nil {
commandContext.SetError(err)
}

if commandContext.OriginalError != nil {
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

logger.MarkSuccess("Cluster deleted!")
return nil
}

func (s *deleteBootstrapClusterForDeleteTask) Name() string {
return "kind-cluster-delete"
}

func (s *deleteBootstrapClusterForDeleteTask) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
return nil, nil
}

func (s *deleteBootstrapClusterForDeleteTask) Checkpoint() *task.CompletedTask {
return nil
}
78 changes: 78 additions & 0 deletions pkg/workflows/management/delete_cluster.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package management

import (
"context"

"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/task"
"github.com/aws/eks-anywhere/pkg/workflows"
)

type deleteManagementCluster struct{}

func (s *deleteManagementCluster) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Deleting management cluster")

err := commandContext.ClusterManager.ResumeEKSAControllerReconcile(ctx, commandContext.BootstrapCluster, commandContext.ClusterSpec, commandContext.Provider)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

err = commandContext.ClusterManager.AddManagedByCLIAnnotationForCluster(ctx, commandContext.BootstrapCluster, commandContext.ClusterSpec, commandContext.Provider)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

err = commandContext.ClusterDeleter.Run(ctx, commandContext.ClusterSpec, *commandContext.BootstrapCluster)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

err = commandContext.Provider.PostClusterDeleteValidate(ctx, commandContext.BootstrapCluster)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

return &cleanupGitRepo{}
}

func (s *deleteManagementCluster) Name() string {
return "delete-management-cluster"
}

func (s *deleteManagementCluster) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
return nil, nil
}

func (s *deleteManagementCluster) Checkpoint() *task.CompletedTask {
return nil
}

type cleanupGitRepo struct{}

func (s *cleanupGitRepo) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Clean up Git Repo")
err := commandContext.GitOpsManager.CleanupGitRepo(ctx, commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectDiagnosticsTask{}
}

return &deleteBootstrapClusterForDeleteTask{}
}

func (s *cleanupGitRepo) Name() string {
return "clean-up-git-repo"
}

func (s *cleanupGitRepo) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
return nil, nil
}

func (s *cleanupGitRepo) Checkpoint() *task.CompletedTask {
return nil
}
41 changes: 41 additions & 0 deletions pkg/workflows/management/delete_create_bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package management

import (
"context"

"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/task"
)

type createBootStrapClusterForDeleteTask struct{}

func (s *createBootStrapClusterForDeleteTask) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Creating new bootstrap cluster")

bootstrapOptions, err := commandContext.Provider.BootstrapClusterOpts(commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return nil
}

bootstrapCluster, err := commandContext.Bootstrapper.CreateBootstrapCluster(ctx, commandContext.ClusterSpec, bootstrapOptions...)
if err != nil {
commandContext.SetError(err)
return nil
}
commandContext.BootstrapCluster = bootstrapCluster

return &installCAPIComponentsForDeleteTask{}
}

func (s *createBootStrapClusterForDeleteTask) Name() string {
return "bootstrap-cluster-for-delete-init"
}

func (s *createBootStrapClusterForDeleteTask) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
return nil, nil
}

func (s *createBootStrapClusterForDeleteTask) Checkpoint() *task.CompletedTask {
return nil
}
Loading

0 comments on commit a590acd

Please sign in to comment.