From 5db8a6937ecaabc622331d69ae70250fe4058b6e Mon Sep 17 00:00:00 2001 From: Berk Dehrioglu Date: Fri, 15 Sep 2023 14:30:09 +0300 Subject: [PATCH] delete awscluster reconciler --- controllers/awscluster_controller.go | 275 ---------------------- controllers/awscluster_controller_test.go | 244 ------------------- 2 files changed, 519 deletions(-) delete mode 100644 controllers/awscluster_controller.go delete mode 100644 controllers/awscluster_controller_test.go diff --git a/controllers/awscluster_controller.go b/controllers/awscluster_controller.go deleted file mode 100644 index e055de19..00000000 --- a/controllers/awscluster_controller.go +++ /dev/null @@ -1,275 +0,0 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "regexp" - "strings" - - awsclientgo "github.com/aws/aws-sdk-go/aws/client" - "github.com/giantswarm/microerror" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - capa "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" - "sigs.k8s.io/cluster-api/util/patch" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "github.com/aws/aws-sdk-go/service/iam/iamiface" - "github.com/go-logr/logr" - - "github.com/giantswarm/capa-iam-operator/pkg/awsclient" - "github.com/giantswarm/capa-iam-operator/pkg/iam" - "github.com/giantswarm/capa-iam-operator/pkg/key" -) - -const ( - IRSASecretSuffix = "irsa-cloudfront" -) - -// AWSClusterReconciler reconciles a AWSCluster object -type AWSClusterReconciler struct { - client.Client - Log logr.Logger - IAMClientFactory func(awsclientgo.ConfigProvider) iamiface.IAMAPI - AWSClient awsclient.AwsClientInterface -} - -// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=secrets/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=core,resources=secrets/finalizers,verbs=update - -func (r *AWSClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := r.Log.WithValues("namespace", req.Namespace, "AWSCluster", req.Name) - logger.Info("Reconciling IRSA roles") - - awsCluster := &capa.AWSCluster{} - err := r.Get(ctx, req.NamespacedName, awsCluster) - if err != nil { - return ctrl.Result{}, errors.WithStack(client.IgnoreNotFound(err)) - } - - if awsCluster.DeletionTimestamp != nil { - return r.reconcileDelete(ctx, logger, awsCluster) - } - - return r.reconcileNormal(ctx, logger, awsCluster) -} - -func (r *AWSClusterReconciler) reconcileNormal(ctx context.Context, logger logr.Logger, awsCluster *capa.AWSCluster) (ctrl.Result, error) { - logger.Info("reconcile normal") - // add finalizer to AWSCluster - if !controllerutil.ContainsFinalizer(awsCluster, key.FinalizerName(iam.IRSARole)) { - patchHelper, err := patch.NewHelper(awsCluster, r.Client) - if err != nil { - return ctrl.Result{}, errors.WithStack(err) - } - controllerutil.AddFinalizer(awsCluster, key.FinalizerName(iam.IRSARole)) - err = patchHelper.Patch(ctx, awsCluster) - if err != nil { - logger.Error(err, "failed to add finalizer on AWSCluster") - return ctrl.Result{}, errors.WithStack(err) - } - logger.Info("successfully added finalizer to AWSCluster", "finalizer_name", key.FinalizerName(iam.IRSARole)) - } - - cm := &corev1.ConfigMap{} - err := r.Get( - ctx, - types.NamespacedName{ - Namespace: awsCluster.Namespace, - Name: fmt.Sprintf("%s-%s", awsCluster.Name, "cluster-values"), - }, - cm) - if err != nil { - logger.Error(err, "Failed to get the cluster-values configmap for cluster") - return ctrl.Result{}, errors.WithStack(err) - } - - if !controllerutil.ContainsFinalizer(cm, key.FinalizerName(iam.IRSARole)) { - patchHelper, err := patch.NewHelper(cm, r.Client) - if err != nil { - return ctrl.Result{}, errors.WithStack(err) - } - controllerutil.AddFinalizer(cm, key.FinalizerName(iam.IRSARole)) - err = patchHelper.Patch(ctx, cm) - if err != nil { - logger.Error(err, "failed to add finalizer to configmap", "configmap", fmt.Sprintf("%s-%s", awsCluster.Name, "cluster-values")) - return ctrl.Result{}, errors.WithStack(err) - } - logger.Info("successfully added finalizer to configmap", "finalizer_name", iam.IRSARole, "configmap", fmt.Sprintf("%s-%s", awsCluster.Name, "cluster-values")) - } - - awsClusterRoleIdentity, err := key.GetAWSClusterRoleIdentity(ctx, r.Client, awsCluster.Spec.IdentityRef.Name) - if err != nil { - logger.Error(err, "could not get AWSClusterRoleIdentity") - return ctrl.Result{}, errors.WithStack(err) - } - - accountID, err := getAWSAccountID(awsClusterRoleIdentity) - if err != nil { - logger.Error(err, "Could not get account ID") - return ctrl.Result{}, errors.WithStack(err) - } - - baseDomain, err := key.GetBaseDomain(ctx, r.Client, awsCluster.Name, awsCluster.Namespace) - if err != nil { - logger.Error(err, "Could not get base domain") - return ctrl.Result{}, errors.WithStack(err) - } - - cloudFrontDomain := key.CloudFrontAlias(baseDomain) - - awsClientSession, err := r.AWSClient.GetAWSClientSession(awsClusterRoleIdentity.Spec.RoleArn, awsCluster.Spec.Region) - if err != nil { - logger.Error(err, "Failed to get aws client session", "cluster_name", awsCluster) - return ctrl.Result{}, errors.WithStack(err) - } - - var iamService *iam.IAMService - { - c := iam.IAMServiceConfig{ - AWSSession: awsClientSession, - ClusterName: awsCluster.Name, - MainRoleName: "-", - RoleType: iam.IRSARole, - Region: awsCluster.Spec.Region, - Log: logger, - IAMClientFactory: r.IAMClientFactory, - } - iamService, err = iam.New(c) - if err != nil { - logger.Error(err, "Failed to generate IAM service") - return ctrl.Result{}, errors.WithStack(err) - } - } - - err = iamService.ReconcileRolesForIRSA(accountID, cloudFrontDomain) - if err != nil { - logger.Error(err, "Unable to reconcile role") - return ctrl.Result{}, errors.WithStack(err) - } - - return ctrl.Result{}, nil -} - -func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, awsCluster *capa.AWSCluster) (ctrl.Result, error) { - logger.Info("reconcile delete") - awsClusterRoleIdentity, err := key.GetAWSClusterRoleIdentity(ctx, r.Client, awsCluster.Spec.IdentityRef.Name) - if err != nil { - logger.Error(err, "could not get AWSClusterRoleIdentity") - return ctrl.Result{}, microerror.Mask(err) - } - awsClientSession, err := r.AWSClient.GetAWSClientSession(awsClusterRoleIdentity.Spec.RoleArn, awsCluster.Spec.Region) - if err != nil { - logger.Error(err, "Failed to get aws client session") - return ctrl.Result{}, errors.WithStack(err) - } - - var iamService *iam.IAMService - { - c := iam.IAMServiceConfig{ - AWSSession: awsClientSession, - ClusterName: awsCluster.Name, - MainRoleName: "-", - RoleType: iam.IRSARole, - Log: logger, - IAMClientFactory: r.IAMClientFactory, - } - iamService, err = iam.New(c) - if err != nil { - logger.Error(err, "Failed to generate IAM service") - } - } - - err = iamService.DeleteRolesForIRSA() - if err != nil { - logger.Error(err, "Unable to reconcile role") - return ctrl.Result{}, err - } - - if controllerutil.ContainsFinalizer(awsCluster, key.FinalizerName(iam.IRSARole)) { - patchHelper, err := patch.NewHelper(awsCluster, r.Client) - if err != nil { - return ctrl.Result{}, err - } - controllerutil.RemoveFinalizer(awsCluster, key.FinalizerName(iam.IRSARole)) - err = patchHelper.Patch(ctx, awsCluster) - if err != nil { - logger.Error(err, "failed to remove finalizer from awsCluster", "finalizer_name", key.FinalizerName(iam.IRSARole), "cluster_name", awsCluster) - return ctrl.Result{}, err - } - logger.Info("successfully removed finalizer from awsCluster", "finalizer_name", key.FinalizerName(iam.IRSARole), "cluster_name", awsCluster) - } - - cm := &corev1.ConfigMap{} - err = r.Get( - ctx, - types.NamespacedName{ - Namespace: awsCluster.Namespace, - Name: fmt.Sprintf("%s-%s", awsCluster.Name, "cluster-values"), - }, - cm) - if err != nil { - logger.Error(err, "Failed to get the cluster-values configmap for cluster") - return ctrl.Result{}, err - } - - if controllerutil.ContainsFinalizer(cm, key.FinalizerName(iam.IRSARole)) { - patchHelper, err := patch.NewHelper(cm, r.Client) - if err != nil { - return ctrl.Result{}, err - } - controllerutil.RemoveFinalizer(cm, key.FinalizerName(iam.IRSARole)) - err = patchHelper.Patch(ctx, cm) - if err != nil { - logger.Error(err, "failed to remove finalizer from configmap") - return ctrl.Result{}, err - } - logger.Info("successfully removed finalizer from configmap", "finalizer_name", iam.IRSARole) - } - return ctrl.Result{}, nil -} - -func getAWSAccountID(awsClusterRoleIdentity *capa.AWSClusterRoleIdentity) (string, error) { - arn := awsClusterRoleIdentity.Spec.RoleArn - if arn == "" || len(strings.TrimSpace(arn)) < 1 { - err := fmt.Errorf("unable to extract ARN from AWSClusterRoleIdentity %s", awsClusterRoleIdentity.Name) - return "", err - } - - re := regexp.MustCompile(`[-]?\d[\d,]*[\.]?[\d{2}]*`) - accountID := re.FindAllString(arn, 1)[0] - - if accountID == "" || len(strings.TrimSpace(accountID)) < 1 { - err := fmt.Errorf("unable to extract AWS account ID from ARN %s", arn) - return "", err - } - - return accountID, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *AWSClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&capa.AWSCluster{}). - Complete(r) -} diff --git a/controllers/awscluster_controller_test.go b/controllers/awscluster_controller_test.go deleted file mode 100644 index 0392900a..00000000 --- a/controllers/awscluster_controller_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package controllers_test - -import ( - "context" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - awsclientupstream "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/iam/iamiface" - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - capa "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" - capi "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "github.com/giantswarm/capa-iam-operator/controllers" - "github.com/giantswarm/capa-iam-operator/pkg/test/mocks" -) - -var _ = Describe("AWSClusterReconciler", func() { - var ( - ctx context.Context - mockCtrl *gomock.Controller - mockAwsClient *mocks.MockAwsClientInterface - mockIAMClient *mocks.MockIAMAPI - reconcileErr error - reconciler *controllers.AWSClusterReconciler - req ctrl.Request - namespace string - sess *session.Session - ) - - SetupNamespaceBeforeAfterEach(&namespace) - - BeforeEach(func() { - logger := zap.New(zap.WriteTo(GinkgoWriter)) - ctx = log.IntoContext(context.Background(), logger) - - mockCtrl = gomock.NewController(GinkgoT()) - - ctx := context.TODO() - - mockAwsClient = mocks.NewMockAwsClientInterface(mockCtrl) - mockIAMClient = mocks.NewMockIAMAPI(mockCtrl) - - reconciler = &controllers.AWSClusterReconciler{ - Client: k8sClient, - Log: ctrl.Log, - AWSClient: mockAwsClient, - IAMClientFactory: func(session awsclientupstream.ConfigProvider) iamiface.IAMAPI { - return mockIAMClient - }, - } - - err := k8sClient.Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-cluster-values", - Namespace: namespace, - }, - Data: map[string]string{ - "values": "baseDomain: test.gaws.gigantic.io\n", - }, - }) - Expect(err).NotTo(HaveOccurred()) - - _ = k8sClient.Create(ctx, &capa.AWSClusterRoleIdentity{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-2", - }, - Spec: capa.AWSClusterRoleIdentitySpec{ - AWSRoleSpec: capa.AWSRoleSpec{ - RoleArn: "arn:aws:iam::012345678901:role/giantswarm-test-capa-controller", - }, - AWSClusterIdentitySpec: capa.AWSClusterIdentitySpec{ - AllowedNamespaces: &capa.AllowedNamespaces{}, - }, - }, - }) - - err = k8sClient.Create(ctx, &capa.AWSCluster{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "cluster.x-k8s.io/cluster-name": "test-cluster", - }, - Name: "test-cluster", - Namespace: namespace, - }, - Spec: capa.AWSClusterSpec{ - IdentityRef: &capa.AWSIdentityReference{ - Name: "test-2", - Kind: "AWSClusterRoleIdentity", - }, - Region: "eu-west-1", - }, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.Create(ctx, &capi.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: namespace, - }, - Spec: capi.ClusterSpec{ - ControlPlaneEndpoint: capi.APIEndpoint{ - Host: "api.testcluster-apiserver-123456789.eu-west-2.elb.amazonaws.com", - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - Expect(namespace).NotTo(BeEmpty()) - req = ctrl.Request{ - NamespacedName: client.ObjectKey{ - Name: "test-cluster", - Namespace: namespace, - }, - } - - sess, err = session.NewSession(&aws.Config{ - Region: aws.String("eu-west-1")}, - ) - Expect(err).NotTo(HaveOccurred()) - - }) - - AfterEach(func() { - mockCtrl.Finish() - }) - - // TODO We create multiple equal policies (`control-plane-test-cluster-policy` - // vs. `irsa-role-test-cluster-policy`. Until we fix that, the test checks - // the current, wrong behavior :( - externalDnsRoleInfoCopy := externalDnsRoleInfo - Expect(externalDnsRoleInfoCopy.ExpectedPolicyName).To(Equal("control-plane-test-cluster-policy")) - externalDnsRoleInfoCopy.ExpectedPolicyName = irsaRoleName - certManagerRoleInfoCopy := certManagerRoleInfo - Expect(certManagerRoleInfoCopy.ExpectedPolicyName).To(Equal("control-plane-test-cluster-policy")) - certManagerRoleInfoCopy.ExpectedPolicyName = irsaRoleName - ALBControllerRoleInfoCopy := ALBControllerRoleInfo - Expect(ALBControllerRoleInfoCopy.ExpectedPolicyName).To(Equal("control-plane-test-cluster-policy")) - ALBControllerRoleInfoCopy.ExpectedPolicyName = irsaRoleName - - expectedRoleStatusesOnSuccess := []RoleInfo{ - certManagerRoleInfoCopy, - externalDnsRoleInfoCopy, - ALBControllerRoleInfoCopy, - } - - expectedIAMTags := []*iam.Tag{ - { - Key: aws.String("capi-iam-controller/owned"), - Value: aws.String(""), - }, - { - Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), - Value: aws.String("owned"), - }, - } - - When("KIAM role was already created by other controller", func() { - BeforeEach(func() { - mockAwsClient.EXPECT().GetAWSClientSession("arn:aws:iam::012345678901:role/giantswarm-test-capa-controller", "eu-west-1").Return(sess, nil) - // Implementation detail: KIAM role gets looked up for each role, therefore `MinTimes(1)` - mockIAMClient.EXPECT().GetRole(&iam.GetRoleInput{ - RoleName: aws.String("test-cluster-IAMManager-Role"), - }).MinTimes(1).Return(&iam.GetRoleOutput{ - Role: &iam.Role{ - Arn: aws.String("arn:aws:iam::999666333:role/test-cluster-IAMManager-Role"), - Tags: expectedIAMTags, - }, - }, nil) - }) - - When("a role does not exist", func() { - BeforeEach(func() { - for _, info := range expectedRoleStatusesOnSuccess { - mockIAMClient.EXPECT().GetRole(&iam.GetRoleInput{ - RoleName: aws.String(info.ExpectedName), - }).Return(nil, awserr.New(iam.ErrCodeNoSuchEntityException, "unit test", nil)) - } - }) - - It("creates the role", func() { - for _, info := range expectedRoleStatusesOnSuccess { - mockIAMClient.EXPECT().CreateRole(&iam.CreateRoleInput{ - AssumeRolePolicyDocument: aws.String(info.ExpectedAssumeRolePolicyDocument), - RoleName: aws.String(info.ExpectedName), - Tags: expectedIAMTags, - }).Return(&iam.CreateRoleOutput{}, nil) - - mockIAMClient.EXPECT().CreateInstanceProfile(&iam.CreateInstanceProfileInput{ - InstanceProfileName: aws.String(info.ExpectedName), - Tags: expectedIAMTags, - }).Return(&iam.CreateInstanceProfileOutput{}, nil) - - mockIAMClient.EXPECT().AddRoleToInstanceProfile(&iam.AddRoleToInstanceProfileInput{ - InstanceProfileName: aws.String(info.ExpectedName), - RoleName: aws.String(info.ExpectedName), - }).Return(&iam.AddRoleToInstanceProfileOutput{}, nil) - - // TODO This is different from `AWSMachineTemplateReconciler`. We should update the policy - // document if and only if it differs from the desired one. - mockIAMClient.EXPECT().UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{ - PolicyDocument: aws.String(info.ExpectedAssumeRolePolicyDocument), - RoleName: aws.String(info.ExpectedName), - }).Return(&iam.UpdateAssumeRolePolicyOutput{}, nil) - - // Implementation detail: instead of storing the ARN, the controller calls `GetRole` multiple times - // from different places. Remove once we don't do this anymore (hence the `MinTimes` call so we - // would notice). - mockIAMClient.EXPECT().GetRole(&iam.GetRoleInput{ - RoleName: aws.String(info.ExpectedName), - }).MinTimes(1).Return(&iam.GetRoleOutput{ - Role: &iam.Role{ - Arn: aws.String(info.ReturnRoleArn), - Tags: expectedIAMTags, - }, - }, nil) - - mockIAMClient.EXPECT().ListRolePolicies(&iam.ListRolePoliciesInput{ - RoleName: aws.String(info.ExpectedName), - }).Return(&iam.ListRolePoliciesOutput{}, nil) - - mockIAMClient.EXPECT().PutRolePolicy(&iam.PutRolePolicyInput{ - PolicyName: aws.String(info.ExpectedPolicyName), - PolicyDocument: aws.String(info.ExpectedPolicyDocument), - RoleName: aws.String(info.ExpectedName), - }).Return(&iam.PutRolePolicyOutput{}, nil) - } - - _, reconcileErr = reconciler.Reconcile(ctx, req) - Expect(reconcileErr).To(BeNil()) - }) - }) - }) -})