From 3f9bd5b9fa8361c7b276cf76fa562fa0f1ceac20 Mon Sep 17 00:00:00 2001 From: Mikhail Fedosin Date: Tue, 12 Sep 2023 19:07:06 +0200 Subject: [PATCH] Allow to configure garbage collector using clusterawsadm --- cmd/clusterawsadm/cmd/gc/configure.go | 86 ++++++++++++++++++++ cmd/clusterawsadm/cmd/gc/gc.go | 1 + cmd/clusterawsadm/gc/gc.go | 41 +++++++++- cmd/clusterawsadm/gc/gc_test.go | 110 +++++++++++++++++++++++++- pkg/annotations/annotations.go | 11 +++ 5 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 cmd/clusterawsadm/cmd/gc/configure.go diff --git a/cmd/clusterawsadm/cmd/gc/configure.go b/cmd/clusterawsadm/cmd/gc/configure.go new file mode 100644 index 0000000000..8c5782b678 --- /dev/null +++ b/cmd/clusterawsadm/cmd/gc/configure.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gc + +import ( + "fmt" + "path/filepath" + + "github.com/spf13/cobra" + "k8s.io/client-go/util/homedir" + + gcproc "sigs.k8s.io/cluster-api-provider-aws/v2/cmd/clusterawsadm/gc" + "sigs.k8s.io/cluster-api/cmd/clusterctl/cmd" +) + +func newConfigureCmd() *cobra.Command { + var ( + clusterName string + namespace string + kubeConfig string + kubeConfigDefault string + gcTasks []string + ) + + if home := homedir.HomeDir(); home != "" { + kubeConfigDefault = filepath.Join(home, ".kube", "config") + } + + newCmd := &cobra.Command{ + Use: "configure", + Short: "Specify what cleanup tasks will be executed on a given cluster", + Long: cmd.LongDesc(` + This command will set what cleanup tasks to execute on the given cluster + during garbage collection (i.e. deleting) when the cluster is + requested to be deleted. Supported values: load-balancer, security-group, target-group. + `), + Example: cmd.Examples(` + # Configure GC for a cluster to delete only load balancers and security groups using existing k8s context + clusterawsadm gc configure --cluster-name=test-cluster --gc-task load-balancer --gc-task security-group + + # Reset GC configuration for a cluster using kubeconfig + clusterawsadm gc configure --cluster-name=test-cluster --kubeconfig=test.kubeconfig + `), + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + proc, err := gcproc.New(gcproc.GCInput{ + ClusterName: clusterName, + Namespace: namespace, + KubeconfigPath: kubeConfig, + }) + if err != nil { + return fmt.Errorf("creating command processor: %w", err) + } + + if err := proc.Configure(cmd.Context(), gcTasks); err != nil { + return fmt.Errorf("configuring garbage collection: %w", err) + } + fmt.Printf("Configuring garbage collection for cluster %s/%s\n", namespace, clusterName) + + return nil + }, + } + + newCmd.Flags().StringVar(&clusterName, "cluster-name", "", "The name of the CAPA cluster") + newCmd.Flags().StringVarP(&namespace, "namespace", "n", "default", "The namespace for the cluster definition") + newCmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfigDefault, "Path to the kubeconfig file to use") + newCmd.Flags().StringSliceVar(&gcTasks, "gc-task", []string{}, "Garbage collection tasks to execute during cluster deletion") + + newCmd.MarkFlagRequired("cluster-name") //nolint: errcheck + + return newCmd +} diff --git a/cmd/clusterawsadm/cmd/gc/gc.go b/cmd/clusterawsadm/cmd/gc/gc.go index d5d95b9f29..0bd0344514 100644 --- a/cmd/clusterawsadm/cmd/gc/gc.go +++ b/cmd/clusterawsadm/cmd/gc/gc.go @@ -36,6 +36,7 @@ func RootCmd() *cobra.Command { newCmd.AddCommand(newEnableCmd()) newCmd.AddCommand(newDisableCmd()) + newCmd.AddCommand(newConfigureCmd()) return newCmd } diff --git a/cmd/clusterawsadm/gc/gc.go b/cmd/clusterawsadm/gc/gc.go index f5a43c432b..046c841be6 100644 --- a/cmd/clusterawsadm/gc/gc.go +++ b/cmd/clusterawsadm/gc/gc.go @@ -19,6 +19,7 @@ package gc import ( "context" "fmt" + "strings" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -104,7 +105,7 @@ func New(input GCInput, opts ...CmdProcessorOption) (*CmdProcessor, error) { // Enable is used to enable external resource garbage collection for a cluster. func (c *CmdProcessor) Enable(ctx context.Context) error { - if err := c.setAnnotationAndPatch(ctx, "true"); err != nil { + if err := c.setAnnotationAndPatch(ctx, infrav1.ExternalResourceGCAnnotation, "true"); err != nil { return fmt.Errorf("setting gc annotation to true: %w", err) } @@ -113,14 +114,42 @@ func (c *CmdProcessor) Enable(ctx context.Context) error { // Disable is used to disable external resource garbage collection for a cluster. func (c *CmdProcessor) Disable(ctx context.Context) error { - if err := c.setAnnotationAndPatch(ctx, "false"); err != nil { + if err := c.setAnnotationAndPatch(ctx, infrav1.ExternalResourceGCAnnotation, "false"); err != nil { return fmt.Errorf("setting gc annotation to false: %w", err) } return nil } -func (c *CmdProcessor) setAnnotationAndPatch(ctx context.Context, annotationValue string) error { +// Configure is used to configure external resource garbage collection for a cluster. +func (c *CmdProcessor) Configure(ctx context.Context, gcTasks []string) error { + supportedGCTasks := []infrav1.GCTask{infrav1.GCTaskLoadBalancer, infrav1.GCTaskTargetGroup, infrav1.GCTaskSecurityGroup} + + for _, gcTask := range gcTasks { + found := false + + for _, supportedGCTask := range supportedGCTasks { + if gcTask == string(supportedGCTask) { + found = true + break + } + } + + if !found { + return fmt.Errorf("unsupported gc task: %s", gcTask) + } + } + + annotationValue := strings.Join(gcTasks, ",") + + if err := c.setAnnotationAndPatch(ctx, infrav1.ExternalResourceGCTasksAnnotation, annotationValue); err != nil { + return fmt.Errorf("setting gc tasks annotation to %s: %w", annotationValue, err) + } + + return nil +} + +func (c *CmdProcessor) setAnnotationAndPatch(ctx context.Context, annotationName, annotationValue string) error { infraObj, err := c.getInfraCluster(ctx) if err != nil { return err @@ -131,7 +160,11 @@ func (c *CmdProcessor) setAnnotationAndPatch(ctx context.Context, annotationValu return fmt.Errorf("creating patch helper: %w", err) } - annotations.Set(infraObj, infrav1.ExternalResourceGCAnnotation, annotationValue) + if annotationValue != "" { + annotations.Set(infraObj, annotationName, annotationValue) + } else { + annotations.Delete(infraObj, annotationName) + } if err := patchHelper.Patch(ctx, infraObj); err != nil { return fmt.Errorf("patching infra cluster with gc annotation: %w", err) diff --git a/cmd/clusterawsadm/gc/gc_test.go b/cmd/clusterawsadm/gc/gc_test.go index 133baa065e..f4e11de3eb 100644 --- a/cmd/clusterawsadm/gc/gc_test.go +++ b/cmd/clusterawsadm/gc/gc_test.go @@ -18,6 +18,7 @@ package gc import ( "context" + "strings" "testing" . "github.com/onsi/gomega" @@ -34,11 +35,13 @@ import ( "sigs.k8s.io/cluster-api/controllers/external" ) +const ( + testClusterName = "test-cluster" +) + func TestEnableGC(t *testing.T) { RegisterTestingT(t) - testClusterName := "test-cluster" - testCases := []struct { name string clusterName string @@ -116,8 +119,6 @@ func TestEnableGC(t *testing.T) { func TestDisableGC(t *testing.T) { RegisterTestingT(t) - testClusterName := "test-cluster" - testCases := []struct { name string clusterName string @@ -186,6 +187,107 @@ func TestDisableGC(t *testing.T) { } } +func TestConfigureGC(t *testing.T) { + RegisterTestingT(t) + + testCases := []struct { + name string + clusterName string + gcTasks []string + existingObjs []client.Object + expectError bool + }{ + { + name: "no capi cluster", + clusterName: testClusterName, + existingObjs: []client.Object{}, + expectError: true, + }, + { + name: "no infra cluster", + clusterName: testClusterName, + existingObjs: newManagedCluster(testClusterName, true), + expectError: true, + }, + { + name: "with managed control plane and no annotation", + clusterName: testClusterName, + existingObjs: newManagedCluster(testClusterName, false), + gcTasks: []string{"load-balancer", "target-group"}, + expectError: false, + }, + { + name: "with awscluster and no annotation", + clusterName: testClusterName, + existingObjs: newUnManagedCluster(testClusterName, false), + gcTasks: []string{"load-balancer", "security-group"}, + expectError: false, + }, + { + name: "with managed control plane and with annotation", + clusterName: testClusterName, + existingObjs: newManagedClusterWithAnnotations(testClusterName, map[string]string{infrav1.ExternalResourceGCTasksAnnotation: "security-group"}), + gcTasks: []string{"load-balancer", "target-group"}, + expectError: false, + }, + { + name: "with awscluster and with annotation", + clusterName: testClusterName, + existingObjs: newUnManagedClusterWithAnnotations(testClusterName, map[string]string{infrav1.ExternalResourceGCTasksAnnotation: "security-group"}), + gcTasks: []string{"load-balancer", "target-group"}, + expectError: false, + }, + { + name: "with awscluster and invalid gc tasks", + clusterName: testClusterName, + existingObjs: newUnManagedCluster(testClusterName, false), + gcTasks: []string{"load-balancer", "INVALID"}, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + input := GCInput{ + ClusterName: tc.clusterName, + Namespace: "default", + } + + fake := newFakeClient(scheme, tc.existingObjs...) + ctx := context.TODO() + + proc, err := New(input, WithClient(fake)) + g.Expect(err).NotTo(HaveOccurred()) + + resErr := proc.Configure(ctx, tc.gcTasks) + if tc.expectError { + g.Expect(resErr).To(HaveOccurred()) + return + } + g.Expect(resErr).NotTo(HaveOccurred()) + + cluster := tc.existingObjs[0].(*clusterv1.Cluster) + ref := cluster.Spec.InfrastructureRef + + obj, err := external.Get(ctx, fake, ref, "default") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(obj).NotTo(BeNil()) + + expected := strings.Join(tc.gcTasks, ",") + annotationVal, hasAnnotation := annotations.Get(obj, infrav1.ExternalResourceGCTasksAnnotation) + + if expected != "" { + g.Expect(hasAnnotation).To(BeTrue()) + g.Expect(annotationVal).To(Equal(expected)) + } else { + g.Expect(hasAnnotation).To(BeFalse()) + } + }) + } +} + func newFakeClient(scheme *runtime.Scheme, objs ...client.Object) client.Client { return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() } diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index d946c64a23..debcd25153 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -42,6 +42,17 @@ func Get(obj metav1.Object, name string) (value string, found bool) { return } +// Delete will delete the supplied annotation. +func Delete(obj metav1.Object, name string) { + annotations := obj.GetAnnotations() + if len(annotations) == 0 { + return + } + + delete(annotations, name) + obj.SetAnnotations(annotations) +} + // Has returns true if the supplied object has the supplied annotation. func Has(obj metav1.Object, name string) bool { annotations := obj.GetAnnotations()