From b2eff2db352ddbb6dbf9b4a35cf5443dd946df1c Mon Sep 17 00:00:00 2001 From: calvix Date: Wed, 1 Nov 2023 08:15:28 +0100 Subject: [PATCH 1/9] capi-add-tags-to-resources --- CHANGELOG.md | 1 + pkg/irsa/capa/capa.go | 22 +++++++++------------- pkg/irsa/eks/eks.go | 7 +++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc4fa49..9f7ad6d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add `global.podSecurityStandards.enforced` value for PSS migration. +- Add AWS tags to all created resources for CAPA and EKS clusters. ## [0.20.0] - 2023-09-21 diff --git a/pkg/irsa/capa/capa.go b/pkg/irsa/capa/capa.go index 470a9115..650ca3f8 100644 --- a/pkg/irsa/capa/capa.go +++ b/pkg/irsa/capa/capa.go @@ -16,7 +16,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - capi "sigs.k8s.io/cluster-api/api/v1beta1" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/giantswarm/irsa-operator/pkg/aws/scope" @@ -95,17 +95,13 @@ func (s *Service) Reconcile(ctx context.Context) error { return err } - // Fetch custom tags from Cluster CR - cluster := &capi.Cluster{} - err = s.Client.Get(ctx, types.NamespacedName{Namespace: s.Scope.ClusterNamespace(), Name: s.Scope.ClusterName()}, cluster) - if apierrors.IsNotFound(err) { - // fallthrough - } else if err != nil { + // Fetch custom tags from AWSCluster CR + awsCluster := &capa.AWSCluster{} + err = s.Client.Get(ctx, types.NamespacedName{Namespace: s.Scope.ClusterNamespace(), Name: s.Scope.ClusterName()}, awsCluster) + if err != nil { return err } - customerTags := key.GetCustomerTags(cluster) - - err = s.S3.CreateTags(s.Scope.BucketName(), customerTags) + err = s.S3.CreateTags(s.Scope.BucketName(), awsCluster.Spec.AdditionalTags) if err != nil { ctrlmetrics.Errors.WithLabelValues(s.Scope.Installation(), s.Scope.AccountID(), s.Scope.ClusterName(), s.Scope.ClusterNamespace()).Inc() s.Scope.Logger().Error(err, "failed to create tags") @@ -122,7 +118,7 @@ func (s *Service) Reconcile(ctx context.Context) error { cloudfrontAliasDomain := s.getCloudFrontAliasDomain() if cloudfrontAliasDomain != "" { // Ensure ACM certificate. - certificateArn, err := s.ACM.EnsureCertificate(cloudfrontAliasDomain, customerTags) + certificateArn, err := s.ACM.EnsureCertificate(cloudfrontAliasDomain, awsCluster.Spec.AdditionalTags) if err != nil { ctrlmetrics.Errors.WithLabelValues(s.Scope.Installation(), s.Scope.AccountID(), s.Scope.ClusterName(), s.Scope.ClusterNamespace()).Inc() s.Scope.Logger().Error(err, "failed to create ACM certificate") @@ -180,7 +176,7 @@ func (s *Service) Reconcile(ctx context.Context) error { cloudfrontCertificateARN = *certificateArn } - distribution, err = s.Cloudfront.EnsureDistribution(cloudfront.DistributionConfig{CustomerTags: customerTags, Aliases: aliases, CertificateArn: cloudfrontCertificateARN}) + distribution, err = s.Cloudfront.EnsureDistribution(cloudfront.DistributionConfig{CustomerTags: awsCluster.Spec.AdditionalTags, Aliases: aliases, CertificateArn: cloudfrontCertificateARN}) if err != nil { ctrlmetrics.Errors.WithLabelValues(s.Scope.Installation(), s.Scope.AccountID(), s.Scope.ClusterName(), s.Scope.ClusterNamespace()).Inc() s.Scope.Logger().Error(err, "failed to create cloudfront distribution") @@ -313,7 +309,7 @@ func (s *Service) Reconcile(ctx context.Context) error { identityProviderURLs = append(identityProviderURLs, util.EnsureHTTPS(*alias)) } - return s.IAM.EnsureOIDCProviders(identityProviderURLs, key.STSUrl(s.Scope.Region()), customerTags) + return s.IAM.EnsureOIDCProviders(identityProviderURLs, key.STSUrl(s.Scope.Region()), awsCluster.Spec.AdditionalTags) } err = backoff.Retry(createOIDCProvider, b) if err != nil { diff --git a/pkg/irsa/eks/eks.go b/pkg/irsa/eks/eks.go index 7765d9eb..30f06d71 100644 --- a/pkg/irsa/eks/eks.go +++ b/pkg/irsa/eks/eks.go @@ -5,7 +5,7 @@ import ( "github.com/giantswarm/microerror" "k8s.io/apimachinery/pkg/types" - capi "sigs.k8s.io/cluster-api/api/v1beta1" + controlplanecapa "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/giantswarm/irsa-operator/pkg/aws/scope" @@ -42,14 +42,13 @@ func (s *Service) Reconcile(ctx context.Context) error { identityProviderURLs := []string{oidcURL} // Fetch custom tags from Cluster CR - cluster := &capi.Cluster{} + cluster := &controlplanecapa.AWSManagedControlPlane{} err = s.Client.Get(ctx, types.NamespacedName{Namespace: s.Scope.ClusterNamespace(), Name: s.Scope.ClusterName()}, cluster) if err != nil { return microerror.Mask(err) } - customerTags := key.GetCustomerTags(cluster) - err = s.IAM.EnsureOIDCProviders(identityProviderURLs, key.STSUrl(s.Scope.Region()), customerTags) + err = s.IAM.EnsureOIDCProviders(identityProviderURLs, key.STSUrl(s.Scope.Region()), cluster.Spec.AdditionalTags) if err != nil { ctrlmetrics.Errors.WithLabelValues(s.Scope.Installation(), s.Scope.AccountID(), s.Scope.ClusterName(), s.Scope.ClusterNamespace()).Inc() s.Scope.Logger().Error(err, "failed to create OIDC provider") From 4f453c9719c9dc447590cb14af6eb8f6a1723f31 Mon Sep 17 00:00:00 2001 From: calvix Date: Wed, 1 Nov 2023 09:03:32 +0100 Subject: [PATCH 2/9] capi-add-tags-to-resources --- pkg/aws/services/s3/bucket.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/aws/services/s3/bucket.go b/pkg/aws/services/s3/bucket.go index 08633caa..d8246859 100644 --- a/pkg/aws/services/s3/bucket.go +++ b/pkg/aws/services/s3/bucket.go @@ -50,10 +50,6 @@ func (s *Service) CreateTags(bucketName string, customerTags map[string]string) Key: aws.String(key.S3TagOrganization), Value: aws.String(util.RemoveOrg(s.scope.ClusterNamespace())), }, - { - Key: aws.String(key.S3TagCluster), - Value: aws.String(s.scope.ClusterName()), - }, { Key: aws.String(fmt.Sprintf(key.S3TagCloudProvider, s.scope.ClusterName())), Value: aws.String("owned"), @@ -65,6 +61,10 @@ func (s *Service) CreateTags(bucketName string, customerTags map[string]string) }, }, } + // if cluster tag is missing add it + if _, ok := customerTags[key.S3TagCluster]; !ok { + customerTags[key.S3TagCluster] = s.scope.ClusterName() + } for k, v := range customerTags { i.Tagging.TagSet = append(i.Tagging.TagSet, &s3.Tag{Key: aws.String(k), Value: aws.String(v)}) From 5f0de0470b8e8e60f6465799f8f5860da51a9ad9 Mon Sep 17 00:00:00 2001 From: calvix Date: Wed, 1 Nov 2023 09:14:02 +0100 Subject: [PATCH 3/9] capi-add-tags-to-resources --- pkg/aws/services/s3/bucket.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/aws/services/s3/bucket.go b/pkg/aws/services/s3/bucket.go index d8246859..616d5d38 100644 --- a/pkg/aws/services/s3/bucket.go +++ b/pkg/aws/services/s3/bucket.go @@ -63,6 +63,9 @@ func (s *Service) CreateTags(bucketName string, customerTags map[string]string) } // if cluster tag is missing add it if _, ok := customerTags[key.S3TagCluster]; !ok { + if customerTags == nil { + customerTags = make(map[string]string) + } customerTags[key.S3TagCluster] = s.scope.ClusterName() } From 60e4bf8c0a42ae7f88a82a48875a99a1c0eeb139 Mon Sep 17 00:00:00 2001 From: calvix Date: Wed, 1 Nov 2023 10:13:18 +0100 Subject: [PATCH 4/9] fix-dsitribution --- pkg/aws/services/acm/acm.go | 11 +++++++---- pkg/aws/services/cloudfront/cloudfront.go | 12 ++++++++++-- pkg/aws/services/s3/bucket.go | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/aws/services/acm/acm.go b/pkg/aws/services/acm/acm.go index d4606d65..d4d4f259 100644 --- a/pkg/aws/services/acm/acm.go +++ b/pkg/aws/services/acm/acm.go @@ -36,10 +36,6 @@ func (s *Service) EnsureCertificate(domain string, customerTags map[string]strin Key: aws.String(key.S3TagOrganization), Value: aws.String(util.RemoveOrg(s.scope.ClusterNamespace())), }, - { - Key: aws.String(key.S3TagCluster), - Value: aws.String(s.scope.ClusterName()), - }, { Key: aws.String(fmt.Sprintf(key.S3TagCloudProvider, s.scope.ClusterName())), Value: aws.String("owned"), @@ -51,6 +47,13 @@ func (s *Service) EnsureCertificate(domain string, customerTags map[string]strin }, ValidationMethod: aws.String(acm.ValidationMethodDns), } + // add cluster tag if missing (this is case for vintage clusters) + if _, ok := customerTags[key.S3TagCluster]; !ok { + if customerTags == nil { + customerTags = make(map[string]string) + } + customerTags[key.S3TagCluster] = s.scope.ClusterName() + } for k, v := range customerTags { tag := &acm.Tag{ diff --git a/pkg/aws/services/cloudfront/cloudfront.go b/pkg/aws/services/cloudfront/cloudfront.go index e900134e..cef898a7 100644 --- a/pkg/aws/services/cloudfront/cloudfront.go +++ b/pkg/aws/services/cloudfront/cloudfront.go @@ -127,6 +127,7 @@ func (s *Service) EnsureDistribution(config DistributionConfig) (*Distribution, // Add internal and customer tags. { + customerTags := config.CustomerTags for k, v := range s.internalTags() { tag := &cloudfront.Tag{ Key: aws.String(k), @@ -135,7 +136,15 @@ func (s *Service) EnsureDistribution(config DistributionConfig) (*Distribution, i.DistributionConfigWithTags.Tags.Items = append(i.DistributionConfigWithTags.Tags.Items, tag) } - for k, v := range config.CustomerTags { + // add cluster tag if missing (this is case for vintage clusters) + if _, ok := customerTags[key.S3TagCluster]; !ok { + if customerTags == nil { + customerTags = make(map[string]string) + } + customerTags[key.S3TagCluster] = s.scope.ClusterName() + } + + for k, v := range customerTags { tag := &cloudfront.Tag{ Key: aws.String(k), Value: aws.String(v), @@ -268,7 +277,6 @@ func (s *Service) findDistribution() (*Distribution, error) { func (s *Service) internalTags() map[string]string { return map[string]string{ key.S3TagOrganization: util.RemoveOrg(s.scope.ClusterNamespace()), - key.S3TagCluster: s.scope.ClusterName(), fmt.Sprintf(key.S3TagCloudProvider, s.scope.ClusterName()): "owned", key.S3TagInstallation: s.scope.Installation(), } diff --git a/pkg/aws/services/s3/bucket.go b/pkg/aws/services/s3/bucket.go index 616d5d38..f57981da 100644 --- a/pkg/aws/services/s3/bucket.go +++ b/pkg/aws/services/s3/bucket.go @@ -61,7 +61,7 @@ func (s *Service) CreateTags(bucketName string, customerTags map[string]string) }, }, } - // if cluster tag is missing add it + // add cluster tag if missing (this is case for vintage clusters) if _, ok := customerTags[key.S3TagCluster]; !ok { if customerTags == nil { customerTags = make(map[string]string) From 5a0e4b6230fbede7b2cbb6fa9218033460100859 Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 2 Nov 2023 12:05:32 +0100 Subject: [PATCH 5/9] fix-comment --- pkg/aws/services/s3/bucket.go | 2 +- pkg/irsa/eks/eks.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/aws/services/s3/bucket.go b/pkg/aws/services/s3/bucket.go index f57981da..e3e1ee58 100644 --- a/pkg/aws/services/s3/bucket.go +++ b/pkg/aws/services/s3/bucket.go @@ -139,6 +139,7 @@ func (s *Service) IsBucketReady(bucketName string) error { } func (s *Service) UpdatePolicy(bucketName, oaiId string) error { + var cloudfrontPolicy = `{ "Version": "2012-10-17", "Id": "PolicyForCloudFrontPrivateContent", @@ -165,7 +166,6 @@ func (s *Service) UpdatePolicy(bucketName, oaiId string) error { } ] }` - t, err := template.New("").Parse(cloudfrontPolicy) if err != nil { return err diff --git a/pkg/irsa/eks/eks.go b/pkg/irsa/eks/eks.go index 30f06d71..e7d22d42 100644 --- a/pkg/irsa/eks/eks.go +++ b/pkg/irsa/eks/eks.go @@ -41,7 +41,7 @@ func (s *Service) Reconcile(ctx context.Context) error { } identityProviderURLs := []string{oidcURL} - // Fetch custom tags from Cluster CR + // Fetch custom tags from AWSManagedControlPlane CR cluster := &controlplanecapa.AWSManagedControlPlane{} err = s.Client.Get(ctx, types.NamespacedName{Namespace: s.Scope.ClusterNamespace(), Name: s.Scope.ClusterName()}, cluster) if err != nil { From c3e94914bab6000597426b071114cfbf0c42d54e Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 2 Nov 2023 17:37:31 +0100 Subject: [PATCH 6/9] whitespace --- pkg/aws/services/s3/bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/aws/services/s3/bucket.go b/pkg/aws/services/s3/bucket.go index e3e1ee58..f57981da 100644 --- a/pkg/aws/services/s3/bucket.go +++ b/pkg/aws/services/s3/bucket.go @@ -139,7 +139,6 @@ func (s *Service) IsBucketReady(bucketName string) error { } func (s *Service) UpdatePolicy(bucketName, oaiId string) error { - var cloudfrontPolicy = `{ "Version": "2012-10-17", "Id": "PolicyForCloudFrontPrivateContent", @@ -166,6 +165,7 @@ func (s *Service) UpdatePolicy(bucketName, oaiId string) error { } ] }` + t, err := template.New("").Parse(cloudfrontPolicy) if err != nil { return err From e98f6f05add3832580f627f346ceba5f1c8b57dd Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 2 Nov 2023 17:39:09 +0100 Subject: [PATCH 7/9] deepcopy --- pkg/aws/services/cloudfront/cloudfront.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/aws/services/cloudfront/cloudfront.go b/pkg/aws/services/cloudfront/cloudfront.go index cef898a7..80d16aea 100644 --- a/pkg/aws/services/cloudfront/cloudfront.go +++ b/pkg/aws/services/cloudfront/cloudfront.go @@ -127,7 +127,11 @@ func (s *Service) EnsureDistribution(config DistributionConfig) (*Distribution, // Add internal and customer tags. { - customerTags := config.CustomerTags + // deep copy the map from config.CustomerTags + var customerTags map[string]string + for k, v := range config.CustomerTags { + customerTags[k] = v + } for k, v := range s.internalTags() { tag := &cloudfront.Tag{ Key: aws.String(k), From 4f43d4fe03929d6fb48246f69b44a3873428412e Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 2 Nov 2023 17:46:17 +0100 Subject: [PATCH 8/9] fix --- pkg/aws/services/cloudfront/cloudfront.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/aws/services/cloudfront/cloudfront.go b/pkg/aws/services/cloudfront/cloudfront.go index 80d16aea..17c38f21 100644 --- a/pkg/aws/services/cloudfront/cloudfront.go +++ b/pkg/aws/services/cloudfront/cloudfront.go @@ -128,7 +128,7 @@ func (s *Service) EnsureDistribution(config DistributionConfig) (*Distribution, // Add internal and customer tags. { // deep copy the map from config.CustomerTags - var customerTags map[string]string + customerTags := make(map[string]string) for k, v := range config.CustomerTags { customerTags[k] = v } From bcb03f00074360f6eb150933e41667ff0ec60f33 Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 2 Nov 2023 18:07:11 +0100 Subject: [PATCH 9/9] fix3 --- pkg/aws/services/cloudfront/cloudfront.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/aws/services/cloudfront/cloudfront.go b/pkg/aws/services/cloudfront/cloudfront.go index 17c38f21..74be42d6 100644 --- a/pkg/aws/services/cloudfront/cloudfront.go +++ b/pkg/aws/services/cloudfront/cloudfront.go @@ -142,9 +142,6 @@ func (s *Service) EnsureDistribution(config DistributionConfig) (*Distribution, // add cluster tag if missing (this is case for vintage clusters) if _, ok := customerTags[key.S3TagCluster]; !ok { - if customerTags == nil { - customerTags = make(map[string]string) - } customerTags[key.S3TagCluster] = s.scope.ClusterName() }