From b9560461c69c0ff1ae67233b0bc462160b0afff0 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 20 Apr 2023 15:09:37 +0200 Subject: [PATCH 01/21] Create tls certificates for audit- and droptailer with secretsmanager --- go.mod | 2 +- go.sum | 11 +- pkg/controller/controlplane/add.go | 10 +- pkg/controller/controlplane/valuesprovider.go | 251 ++++++++---------- pkg/controller/worker/actuator.go | 2 - 5 files changed, 130 insertions(+), 146 deletions(-) diff --git a/go.mod b/go.mod index 8b0e6d68e..ed7df4d8c 100644 --- a/go.mod +++ b/go.mod @@ -167,7 +167,7 @@ require ( ) replace ( - github.com/gardener/gardener => github.com/gardener/gardener v1.44.6 + github.com/gardener/gardener => github.com/gardener/gardener v1.46.3 k8s.io/api => k8s.io/api v0.23.2 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.2 diff --git a/go.sum b/go.sum index 9fe4a1979..b48d1bd1d 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/gardener/etcd-druid v0.12.3/go.mod h1:EJF6z4Ghv2FGUe1UzZWOEF1MxCA186f github.com/gardener/external-dns-management v0.7.18/go.mod h1:oHhauLQ3/sop0c1urS6n304Wqv/WM4me0geLn9nTAcY= github.com/gardener/external-dns-management v0.12.5 h1:OvnupQuTZDzFSSuEKFKXx51NSEuI3gWOcZJBtCbqzSU= github.com/gardener/external-dns-management v0.12.5/go.mod h1:lFDmJJYumx9wOP8sP0nkucOzQ9v/cVgLJcLX90H/K50= -github.com/gardener/gardener v1.44.6 h1:8NpdiI27P/lC2PRZXdumi8caQe8B5qxL7gtutZHuQpE= -github.com/gardener/gardener v1.44.6/go.mod h1:y8f1w9u44YuDAowo3PWmR79LhUv4TeTTb/1dhAFbKzc= +github.com/gardener/gardener v1.46.3 h1:sR9b73Nr7SieGksWrqvNCa7dwllgJXa6ryH7I0YUScM= +github.com/gardener/gardener v1.46.3/go.mod h1:BsQ9s0Ms5rU1IAS1doVceBGj4kNBhSMQKaDB4a2ba4k= github.com/gardener/gardener-extension-networking-calico v1.9.1 h1:dDHShP9O7goJLb+tMnmftv35bOUVeMl9L/0WQYXux2k= github.com/gardener/gardener-extension-networking-calico v1.9.1/go.mod h1:BloF9rbMbFgV10HPJrrXThKXzSdxmsnJjvugpQ55cq0= github.com/gardener/gardener-extension-networking-cilium v1.18.0 h1:LNBMqVAkltHBDkP+C5Vq/dFgve/YOG8MIvTJJuWWCtU= @@ -255,7 +255,6 @@ github.com/gardener/gardener-extension-networking-cilium v1.18.0/go.mod h1:bXE/C github.com/gardener/gardener-resource-manager v0.13.1/go.mod h1:0No/XttYRUwDn5lSppq9EqlKdo/XJQ44aCZz5BVu3Vw= github.com/gardener/hvpa-controller v0.2.5/go.mod h1:rjsb3BPKJFMluudZ8/bhCCDQfFCF/0Um+rzXQI/MmfI= github.com/gardener/hvpa-controller v0.3.1/go.mod h1:rjsb3BPKJFMluudZ8/bhCCDQfFCF/0Um+rzXQI/MmfI= -github.com/gardener/hvpa-controller/api v0.4.0/go.mod h1:QQl3ELkCaki+8RhXl0FZMfvnm0WCGwGJlGmrxJj6lvM= github.com/gardener/hvpa-controller/api v0.5.0 h1:f4F3O7YUrenwh4S3TgPREPiB287JjjUiUL18OqPLyAA= github.com/gardener/hvpa-controller/api v0.5.0/go.mod h1:QQl3ELkCaki+8RhXl0FZMfvnm0WCGwGJlGmrxJj6lvM= github.com/gardener/machine-controller-manager v0.41.0/go.mod h1:43OABkCemMS6b35z3OprbfaT3p2HxKAZkJekjCO2T48= @@ -717,7 +716,7 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1505,11 +1504,11 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= istio.io/api v0.0.0-20210520012029-891c0c12abfd/go.mod h1:nsSFw1LIMmGL7r/+6fJI6FxeG/UGlLxRK8bkojIvBVs= -istio.io/api v0.0.0-20211118170605-3f0f902cdfd1/go.mod h1:lavaUNsnT7RGyMFNOGgV5XvOgP3fkTSZkxP/0H/ISt4= +istio.io/api v0.0.0-20220304035241-8c47cbbea144/go.mod h1:lavaUNsnT7RGyMFNOGgV5XvOgP3fkTSZkxP/0H/ISt4= istio.io/api v0.0.0-20220512181135-e8ec1e1d89de h1:xYxUL/LyuJ5yb0o/d2GXseYjx9HIBpLHLArp2Zd8d+M= istio.io/api v0.0.0-20220512181135-e8ec1e1d89de/go.mod h1:00myJeQGWma4Y5pboJ+MM4P2uqEWulKA1duC8kYN5Wo= istio.io/client-go v1.10.1/go.mod h1:xiCalTDM2HqP1KGxMOt+OOkqCLOI6QrwQMb95cSWw9U= -istio.io/client-go v1.12.0/go.mod h1:Y46Rc0vTVHogmIXnGMCsb19Bc0XIMhOEnZUr+5ZxMmo= +istio.io/client-go v1.12.5/go.mod h1:rKTRGbzRHY/lb3VCCIF4wwTwvkW000acWEcseSy3V9M= istio.io/client-go v1.14.0 h1:KKXMnxXx3U2866OP8FBYlJhjKdI3yIUQnt8L6hSzDHE= istio.io/client-go v1.14.0/go.mod h1:C7K0CKQlvY84yQKkZhxQbD1riqvnsgXJm3jF5GOmzNg= istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f/go.mod h1:6BwTZRNbWS570wHX/uR1Wqk5e0157TofTAUMzT7N4+s= diff --git a/pkg/controller/controlplane/add.go b/pkg/controller/controlplane/add.go index 9402c0ebb..3b7e13e8d 100644 --- a/pkg/controller/controlplane/add.go +++ b/pkg/controller/controlplane/add.go @@ -37,9 +37,13 @@ type AddOptions struct { // The opts.Reconciler is being set with a newly instantiated actuator. func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) error { return controlplane.Add(mgr, controlplane.AddArgs{ - Actuator: genericactuator.NewActuator(metal.Name, controlPlaneSecrets, shootAccessSecretsFunc, nil, nil, nil, nil, configChart, controlPlaneChart, cpShootChart, nil, - storageClassChart, nil, NewValuesProvider(logger, opts.ControllerConfig), extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot), - imagevector.ImageVector(), "", opts.ShootWebhooks, mgr.GetWebhookServer().Port, logger), + + Actuator: genericactuator.NewActuator(metal.Name, + secretConfigsFunc, shootAccessSecretsFunc, nil, nil, + configChart, controlPlaneChart, cpShootChart, nil, storageClassChart, nil, + NewValuesProvider(logger, opts.ControllerConfig), extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot), + imagevector.ImageVector(), "", opts.ShootWebhooks, mgr.GetWebhookServer().Port, logger, + ), ControllerOptions: opts.Controller, Predicates: controlplane.DefaultPredicates(opts.IgnoreOperationAnnotation), Type: metal.Type, diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 0b2974858..b3d560b04 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -18,7 +18,6 @@ import ( "github.com/metal-stack/metal-lib/pkg/tag" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - gardenerkubernetes "github.com/gardener/gardener/pkg/client/kubernetes" durosv1 "github.com/metal-stack/duros-controller/api/v1" firewallv1 "github.com/metal-stack/firewall-controller/api/v1" @@ -43,6 +42,7 @@ import ( policyv1beta1 "k8s.io/api/policy/v1beta1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -56,9 +56,11 @@ import ( v1alpha1constants "github.com/gardener/gardener/pkg/apis/core/v1alpha1/constants" + extensionssecretsmanager "github.com/gardener/gardener/extensions/pkg/util/secret/manager" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/utils/chart" "github.com/gardener/gardener/pkg/utils/secrets" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" "github.com/go-logr/logr" @@ -69,38 +71,39 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var controlPlaneSecrets = &secrets.Secrets{ - CertificateSecretConfigs: map[string]*secrets.CertificateSecretConfig{ - v1alpha1constants.SecretNameCACluster: { - Name: v1alpha1constants.SecretNameCACluster, - CommonName: "kubernetes", - CertType: secrets.CACert, +func secretConfigsFunc(namespace string) []extensionssecretsmanager.SecretConfigWithOptions { + return []extensionssecretsmanager.SecretConfigWithOptions{ + { + Config: &secrets.CertificateSecretConfig{ + Name: v1alpha1constants.SecretNameCACluster, + CommonName: "kubernetes", + CertType: secrets.CACert, + }, + Options: []secretsmanager.GenerateOption{secretsmanager.Persist()}, }, - }, - SecretConfigsFunc: func(cas map[string]*secrets.Certificate, clusterName string) []secrets.ConfigInterface { - return []secrets.ConfigInterface{ - &secrets.ControlPlaneSecretConfig{ - Name: metal.CloudControllerManagerServerName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.CloudControllerManagerServerName, - CommonName: metal.CloudControllerManagerDeploymentName, - DNSNames: kutil.DNSNamesForService(metal.CloudControllerManagerDeploymentName, clusterName), - CertType: secrets.ServerCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.CloudControllerManagerServerName, + CommonName: metal.CloudControllerManagerDeploymentName, + DNSNames: kutil.DNSNamesForService(metal.CloudControllerManagerDeploymentName, namespace), + CertType: secrets.ServerCert, + SkipPublishingCACertificate: true, }, - &secrets.ControlPlaneSecretConfig{ - Name: metal.FirewallControllerManagerDeploymentName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.FirewallControllerManagerDeploymentName, - CommonName: metal.FirewallControllerManagerDeploymentName, - DNSNames: kutil.DNSNamesForService(metal.FirewallControllerManagerDeploymentName, clusterName), - CertType: secrets.ServerCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(v1alpha1constants.SecretNameCACluster)}, + }, + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.FirewallControllerManagerDeploymentName, + CommonName: metal.FirewallControllerManagerDeploymentName, + DNSNames: kutil.DNSNamesForService(metal.FirewallControllerManagerDeploymentName, namespace), + CertType: secrets.ServerCert, + SkipPublishingCACertificate: true, }, - } - }, + // use current CA for signing server cert to prevent mismatches when dropping the old CA from the webhook + // config in phase Completing + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(v1alpha1constants.SecretNameCACluster, secretsmanager.UseCurrentCA)}, + }, + } } func shootAccessSecretsFunc(namespace string) []*gutil.ShootAccessSecret { @@ -419,9 +422,10 @@ func (vp *valuesProvider) GetControlPlaneChartValues( ctx context.Context, cp *extensionsv1alpha1.ControlPlane, cluster *extensionscontroller.Cluster, + secretsReader secretsmanager.Reader, checksums map[string]string, scaledDown bool, -) (map[string]interface{}, error) { +) (map[string]any, error) { infrastructureConfig := &apismetal.InfrastructureConfig{} if _, _, err := vp.Decoder().Decode(cluster.Shoot.Spec.Provider.InfrastructureConfig.Raw, nil, infrastructureConfig); err != nil { return nil, fmt.Errorf("could not decode providerConfig of infrastructure %w", err) @@ -537,12 +541,14 @@ func merge(target map[string]interface{}, sources ...map[string]interface{}) { func (vp *valuesProvider) GetControlPlaneExposureChartValues( ctx context.Context, cp *extensionsv1alpha1.ControlPlane, - cluster *extensionscontroller.Cluster, m map[string]string) (map[string]interface{}, error) { + cluster *extensionscontroller.Cluster, + secretsReader secretsmanager.Reader, + checksums map[string]string) (map[string]interface{}, error) { return nil, nil } // GetControlPlaneShootChartValues returns the values for the control plane shoot chart applied by the generic actuator. -func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, cp *extensionsv1alpha1.ControlPlane, cluster *extensionscontroller.Cluster, checksums map[string]string) (map[string]interface{}, error) { +func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, cp *extensionsv1alpha1.ControlPlane, cluster *extensionscontroller.Cluster, secretsReader secretsmanager.Reader, checksums map[string]string) (map[string]interface{}, error) { infrastructureConfig := &apismetal.InfrastructureConfig{} if _, _, err := vp.Decoder().Decode(cluster.Shoot.Spec.Provider.InfrastructureConfig.Raw, nil, infrastructureConfig); err != nil { return nil, fmt.Errorf("could not decode providerConfig of infrastructure %w", err) @@ -594,14 +600,13 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, c return nil, err } + // FIXME stefan what to do here if !extensionscontroller.IsHibernated(cluster) { - if validation.ClusterAuditEnabled(&vp.controllerConfig, cpConfig) { - if err := vp.deployControlPlaneShootAudittailerCerts(ctx, cluster); err != nil { - vp.logger.Error(err, "error deploying audittailer certs") - } + if err := vp.deploySecretsToShoot(ctx, cluster, metal.AudittailerNamespace, vp.audittailerSecretConfigs); err != nil { + vp.logger.Error(err, "error deploying audittailer certs") } - if err := vp.deployControlPlaneShootDroptailerCerts(ctx, cluster); err != nil { + if err := vp.deploySecretsToShoot(ctx, cluster, metal.DroptailerNamespace, vp.droptailerSecretConfigs); err != nil { vp.logger.Error(err, "error deploying droptailer certs") } } @@ -845,133 +850,111 @@ func (vp *valuesProvider) signFirewallValues(ctx context.Context, namespace stri return nil } -func (vp *valuesProvider) deployControlPlaneShootAudittailerCerts(ctx context.Context, cluster *extensionscontroller.Cluster) error { - // TODO: There is actually no nice way to deploy the certs into the shoot when we want to use - // the certificate helper functions from Gardener itself... - // Maybe we can find a better solution? This is actually only for chart values... +func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager.SecretConfigWithOptions { + if !vp.controllerConfig.ClusterAudit.Enabled { + return nil + } - wanted := &secrets.Secrets{ - CertificateSecretConfigs: map[string]*secrets.CertificateSecretConfig{ - v1alpha1constants.SecretNameCACluster: { - Name: v1alpha1constants.SecretNameCACluster, - CommonName: "kubernetes", + return []extensionssecretsmanager.SecretConfigWithOptions{ + { + Config: &secrets.CertificateSecretConfig{ + Name: "ca-provider-metal-audittailer", + CommonName: "ca-provider-metal-audittailer", CertType: secrets.CACert, }, + Options: []secretsmanager.GenerateOption{secretsmanager.Persist()}, }, - SecretConfigsFunc: func(cas map[string]*secrets.Certificate, clusterName string) []secrets.ConfigInterface { - return []secrets.ConfigInterface{ - &secrets.ControlPlaneSecretConfig{ - Name: metal.AudittailerClientSecretName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.AudittailerClientSecretName, - CommonName: "audittailer", - DNSNames: []string{"audittailer"}, - Organization: []string{"audittailer-client"}, - CertType: secrets.ClientCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, - }, - &secrets.ControlPlaneSecretConfig{ - Name: metal.AudittailerServerSecretName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.AudittailerServerSecretName, - CommonName: "audittailer", - DNSNames: []string{"audittailer"}, - Organization: []string{"audittailer-server"}, - CertType: secrets.ServerCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, - }, - } + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.AudittailerClientSecretName, + CommonName: "audittailer", + DNSNames: []string{"audittailer"}, + Organization: []string{"audittailer-client"}, + CertType: secrets.ClientCert, + SkipPublishingCACertificate: true, + }, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, + }, + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.AudittailerServerSecretName, + CommonName: "audittailer", + DNSNames: []string{"audittailer"}, + Organization: []string{"audittailer-server"}, + CertType: secrets.ServerCert, + SkipPublishingCACertificate: true, + }, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, }, } - - return vp.deploySecretsToShoot(ctx, cluster, metal.AudittailerNamespace, wanted) } -func (vp *valuesProvider) deployControlPlaneShootDroptailerCerts(ctx context.Context, cluster *extensionscontroller.Cluster) error { - // TODO: There is actually no nice way to deploy the certs into the shoot when we want to use - // the certificate helper functions from Gardener itself... - // Maybe we can find a better solution? This is actually only for chart values... - - wanted := &secrets.Secrets{ - CertificateSecretConfigs: map[string]*secrets.CertificateSecretConfig{ - v1alpha1constants.SecretNameCACluster: { - Name: v1alpha1constants.SecretNameCACluster, - CommonName: "kubernetes", +func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.SecretConfigWithOptions { + return []extensionssecretsmanager.SecretConfigWithOptions{ + { + Config: &secrets.CertificateSecretConfig{ + Name: "ca-provider-metal-droptailer", + CommonName: "ca-provider-metal-droptailer", CertType: secrets.CACert, }, + Options: []secretsmanager.GenerateOption{secretsmanager.Persist()}, }, - SecretConfigsFunc: func(cas map[string]*secrets.Certificate, clusterName string) []secrets.ConfigInterface { - return []secrets.ConfigInterface{ - &secrets.ControlPlaneSecretConfig{ - Name: metal.DroptailerClientSecretName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.DroptailerClientSecretName, - CommonName: "droptailer", - DNSNames: []string{"droptailer"}, - Organization: []string{"droptailer-client"}, - CertType: secrets.ClientCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, - }, - &secrets.ControlPlaneSecretConfig{ - Name: metal.DroptailerServerSecretName, - CertificateSecretConfig: &secrets.CertificateSecretConfig{ - Name: metal.DroptailerServerSecretName, - CommonName: "droptailer", - DNSNames: []string{"droptailer"}, - Organization: []string{"droptailer-server"}, - CertType: secrets.ServerCert, - SigningCA: cas[v1alpha1constants.SecretNameCACluster], - }, - }, - } + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.DroptailerClientSecretName, + CommonName: "droptailer", + DNSNames: []string{"droptailer"}, + Organization: []string{"droptailer-client"}, + CertType: secrets.ClientCert, + SkipPublishingCACertificate: true, + }, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, + }, + { + Config: &secrets.CertificateSecretConfig{ + Name: metal.DroptailerServerSecretName, + CommonName: "droptailer", + DNSNames: []string{"droptailer"}, + Organization: []string{"droptailer-server"}, + CertType: secrets.ServerCert, + SkipPublishingCACertificate: true, + }, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, }, } - - return vp.deploySecretsToShoot(ctx, cluster, metal.DroptailerNamespace, wanted) } -func (vp *valuesProvider) deploySecretsToShoot(ctx context.Context, cluster *extensionscontroller.Cluster, namespace string, wanted *secrets.Secrets) error { +func (vp *valuesProvider) deploySecretsToShoot(ctx context.Context, cluster *extensionscontroller.Cluster, namespace string, secretConfigsFn func() []extensionssecretsmanager.SecretConfigWithOptions) error { shootConfig, _, err := util.NewClientForShoot(ctx, vp.Client(), cluster.ObjectMeta.Name, client.Options{}) if err != nil { return fmt.Errorf("could not create shoot client %w", err) } - cs, err := kubernetes.NewForConfig(shootConfig) + c, err := client.New(shootConfig, client.Options{}) if err != nil { return fmt.Errorf("could not create shoot kubernetes client %w", err) } - gcs, err := gardenerkubernetes.NewWithConfig(gardenerkubernetes.WithRESTConfig(shootConfig)) - if err != nil { - return fmt.Errorf("could not create shoot Gardener client %w", err) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, } - - _, err = cs.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { + return nil + }) if err != nil { - if apierrors.IsNotFound(err) { - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - }, - } - _, err := cs.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("could not create namespace %w", err) - } - } else { - return fmt.Errorf("could not search for existence of namespace %w", err) - } + return fmt.Errorf("could not ensure namespace: %w", err) } - _, err = wanted.Deploy(ctx, cs, gcs, namespace) + manager, err := secretsmanager.New(ctx, vp.logger.WithName("shoot-secrets-manager"), clock.RealClock{}, c, namespace, metal.Type+"-provider-shoot-controlplane", nil) if err != nil { - return fmt.Errorf("could not deploy secrets to shoot cluster %w", err) + return fmt.Errorf("unable to create secrets manager: %w", err) } - return nil + _, err = extensionssecretsmanager.GenerateAllSecrets(ctx, manager, secretConfigsFn()) + + return err } // getSecret returns the secret with the given namespace/secretName diff --git a/pkg/controller/worker/actuator.go b/pkg/controller/worker/actuator.go index 9e638f5dc..56cb6613e 100644 --- a/pkg/controller/worker/actuator.go +++ b/pkg/controller/worker/actuator.go @@ -56,8 +56,6 @@ func NewActuator(machineImages []config.MachineImage, controllerConfig config.Co mcmShootChart, imagevector.ImageVector(), extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot), - true, - true, ) } From a66d5029051463cccc9cd5533c5209bad1ffe667 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 20 Apr 2023 15:22:01 +0200 Subject: [PATCH 02/21] Keep CA --- pkg/controller/controlplane/valuesprovider.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index b3d560b04..2b8f59798 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -871,7 +871,7 @@ func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager. DNSNames: []string{"audittailer"}, Organization: []string{"audittailer-client"}, CertType: secrets.ClientCert, - SkipPublishingCACertificate: true, + SkipPublishingCACertificate: false, }, Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, }, @@ -882,7 +882,7 @@ func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager. DNSNames: []string{"audittailer"}, Organization: []string{"audittailer-server"}, CertType: secrets.ServerCert, - SkipPublishingCACertificate: true, + SkipPublishingCACertificate: false, }, Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, }, @@ -906,7 +906,7 @@ func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.S DNSNames: []string{"droptailer"}, Organization: []string{"droptailer-client"}, CertType: secrets.ClientCert, - SkipPublishingCACertificate: true, + SkipPublishingCACertificate: false, }, Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, }, @@ -917,7 +917,7 @@ func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.S DNSNames: []string{"droptailer"}, Organization: []string{"droptailer-server"}, CertType: secrets.ServerCert, - SkipPublishingCACertificate: true, + SkipPublishingCACertificate: false, }, Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, }, From 9146513b55e4046e46865c0f8d8a8b2b9c5c4ce7 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 08:40:02 +0200 Subject: [PATCH 03/21] use new secrets in droptailer/audittailer --- .../templates/audittailer.yaml | 6 +-- .../templates/firewall/droptailer.yaml | 9 +--- .../internal/shoot-control-plane/values.yaml | 6 +++ pkg/apis/metal/helper/secrets.go | 12 ++++++ pkg/controller/controlplane/valuesprovider.go | 41 ++++++++++++++++--- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/charts/internal/shoot-control-plane/templates/audittailer.yaml b/charts/internal/shoot-control-plane/templates/audittailer.yaml index 79f7c5eae..f6e92d742 100644 --- a/charts/internal/shoot-control-plane/templates/audittailer.yaml +++ b/charts/internal/shoot-control-plane/templates/audittailer.yaml @@ -63,7 +63,7 @@ spec: name: audittailer-config - name: fluentd-certs secret: - secretName: audittailer-server + secretName: {{ .Values.audittailer.secretName }} - name: fluentbuffer emptyDir: {} --- @@ -82,8 +82,8 @@ data: bind 0.0.0.0 ca_path /fluentd/etc/ssl/ca.crt - cert_path /fluentd/etc/ssl/audittailer-server.crt - private_key_path /fluentd/etc/ssl/audittailer-server.key + cert_path /fluentd/etc/ssl/tls.crt + private_key_path /fluentd/etc/ssl/tls.key client_cert_auth true diff --git a/charts/internal/shoot-control-plane/templates/firewall/droptailer.yaml b/charts/internal/shoot-control-plane/templates/firewall/droptailer.yaml index 3345fa475..7cd378011 100644 --- a/charts/internal/shoot-control-plane/templates/firewall/droptailer.yaml +++ b/charts/internal/shoot-control-plane/templates/firewall/droptailer.yaml @@ -62,11 +62,4 @@ spec: volumes: - name: droptailer-server secret: - secretName: droptailer-server - items: - - key: droptailer-server.key - path: tls.key - - key: droptailer-server.crt - path: tls.crt - - key: ca.crt - path: ca.crt + secretName: {{ .Values.droptailer.secretName }} diff --git a/charts/internal/shoot-control-plane/values.yaml b/charts/internal/shoot-control-plane/values.yaml index 7c2000a9b..ace6fc545 100644 --- a/charts/internal/shoot-control-plane/values.yaml +++ b/charts/internal/shoot-control-plane/values.yaml @@ -39,3 +39,9 @@ restrictEgress: matchPattern: '*.a-name.org' protocol: TCP port: 443 + +droptailer: + secretName: 'droptailer-server' + +audittailer: + secretName: 'audittailer-server' diff --git a/pkg/apis/metal/helper/secrets.go b/pkg/apis/metal/helper/secrets.go index bebeb3ed3..646b9cc14 100644 --- a/pkg/apis/metal/helper/secrets.go +++ b/pkg/apis/metal/helper/secrets.go @@ -39,6 +39,18 @@ func GetLatestCABundle(ctx context.Context, c client.Client, namespace string) ( return getLatestIssuedSecret(secretList.Items) } +func GetLatestSecret(ctx context.Context, c client.Client, namespace string,name string) (*corev1.Secret, error) { + secretList := &corev1.SecretList{} + if err := c.List(ctx, secretList, client.InNamespace(namespace), client.MatchingLabels{ + secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, + secretsmanager.LabelKeyName: name, + }); err != nil { + return nil, err + } + + return getLatestIssuedSecret(secretList.Items) +} + // getLatestIssuedSecret returns the secret with the "issued-at-time" label that represents the latest point in time func getLatestIssuedSecret(secrets []corev1.Secret) (*corev1.Secret, error) { if len(secrets) == 0 { diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 2b8f59798..73d195fbb 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -594,12 +594,6 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, c return nil, err } - values, err := vp.getControlPlaneShootChartValues(ctx, metalControlPlane, cpConfig, cluster, nws, infrastructure, infrastructureConfig, mclient) - if err != nil { - vp.logger.Error(err, "Error getting shoot control plane chart values") - return nil, err - } - // FIXME stefan what to do here if !extensionscontroller.IsHibernated(cluster) { if err := vp.deploySecretsToShoot(ctx, cluster, metal.AudittailerNamespace, vp.audittailerSecretConfigs); err != nil { @@ -611,6 +605,12 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, c } } + values, err := vp.getControlPlaneShootChartValues(ctx, metalControlPlane, cpConfig, cluster, nws, infrastructure, infrastructureConfig, mclient) + if err != nil { + vp.logger.Error(err, "Error getting shoot control plane chart values") + return nil, err + } + return values, nil } @@ -684,6 +684,15 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, m }) } + droptailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.DroptailerNamespace, metal.DroptailerServerSecretName) + if err != nil { + return nil, fmt.Errorf("secret %q not found", metal.DroptailerServerSecretName) + } + audittailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.AudittailerNamespace, metal.AudittailerServerSecretName) + if err != nil { + return nil, fmt.Errorf("secret %q not found", metal.AudittailerServerSecretName) + } + values := map[string]any{ "kubernetesVersion": cluster.Shoot.Spec.Kubernetes.Version, "apiserverIPs": apiserverIPs, @@ -699,6 +708,12 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, m "apiServerIngressDomain": "api." + *cluster.Shoot.Spec.DNS.Domain, "destinations": egressDestinations, }, + "droptailer": map[string]any{ + "secretName": droptailerServerSecret.Name, + }, + "audittailer": map[string]any{ + "secretName": audittailerServerSecret.Name, + }, } if vp.controllerConfig.Storage.Duros.Enabled { @@ -957,6 +972,20 @@ func (vp *valuesProvider) deploySecretsToShoot(ctx context.Context, cluster *ext return err } +func (vp *valuesProvider) getSecretFromShoot(ctx context.Context, cluster *extensionscontroller.Cluster, namespace string, name string) (*corev1.Secret, error) { + shootConfig, _, err := util.NewClientForShoot(ctx, vp.Client(), cluster.ObjectMeta.Name, client.Options{}) + if err != nil { + return nil, fmt.Errorf("could not create shoot client %w", err) + } + + c, err := client.New(shootConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("could not create shoot kubernetes client %w", err) + } + + return helper.GetLatestSecret(ctx, c, namespace, name) +} + // getSecret returns the secret with the given namespace/secretName func (vp *valuesProvider) getSecret(ctx context.Context, namespace string, secretName string) (*corev1.Secret, error) { key := kutil.Key(namespace, secretName) From a16df644d1b5daac3522b51001251dbc53c8aeb2 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 09:25:07 +0200 Subject: [PATCH 04/21] allow list secrets for auditforwarder --- charts/internal/shoot-control-plane/templates/audittailer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/internal/shoot-control-plane/templates/audittailer.yaml b/charts/internal/shoot-control-plane/templates/audittailer.yaml index f6e92d742..b33dd2ae9 100644 --- a/charts/internal/shoot-control-plane/templates/audittailer.yaml +++ b/charts/internal/shoot-control-plane/templates/audittailer.yaml @@ -126,6 +126,7 @@ rules: - secrets verbs: - get + - list --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 From d76336b472e93245b944f6e0714cd3be81ff4bde Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 09:52:41 +0200 Subject: [PATCH 05/21] new secrets have different file names --- pkg/webhook/controlplane/ensurer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index e74b80b03..63b1d745b 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -289,11 +289,11 @@ var ( }, { Name: "AUDIT_TLS_CRT_FILE", - Value: "audittailer-client.crt", + Value: "tls.crt", }, { Name: "AUDIT_TLS_KEY_FILE", - Value: "audittailer-client.key", + Value: "tls.key", }, { Name: "AUDIT_TLS_VHOST", From 3095ce471f22610c2547d3e53ffa4608d645b00f Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 10:37:33 +0200 Subject: [PATCH 06/21] linter fix --- pkg/apis/metal/helper/secrets.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/apis/metal/helper/secrets.go b/pkg/apis/metal/helper/secrets.go index 646b9cc14..532092a7f 100644 --- a/pkg/apis/metal/helper/secrets.go +++ b/pkg/apis/metal/helper/secrets.go @@ -39,11 +39,11 @@ func GetLatestCABundle(ctx context.Context, c client.Client, namespace string) ( return getLatestIssuedSecret(secretList.Items) } -func GetLatestSecret(ctx context.Context, c client.Client, namespace string,name string) (*corev1.Secret, error) { +func GetLatestSecret(ctx context.Context, c client.Client, namespace string, name string) (*corev1.Secret, error) { secretList := &corev1.SecretList{} if err := c.List(ctx, secretList, client.InNamespace(namespace), client.MatchingLabels{ - secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, - secretsmanager.LabelKeyName: name, + secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, + secretsmanager.LabelKeyName: name, }); err != nil { return nil, err } From 9474099d638d7151d825984a940346c0af073b96 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 14:33:20 +0200 Subject: [PATCH 07/21] do not return error if secret is not yet found --- pkg/controller/controlplane/valuesprovider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 73d195fbb..5560146d3 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -686,11 +686,11 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, m droptailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.DroptailerNamespace, metal.DroptailerServerSecretName) if err != nil { - return nil, fmt.Errorf("secret %q not found", metal.DroptailerServerSecretName) + vp.logger.Info("secret not found", "secret", metal.DroptailerServerSecretName) } audittailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.AudittailerNamespace, metal.AudittailerServerSecretName) if err != nil { - return nil, fmt.Errorf("secret %q not found", metal.AudittailerServerSecretName) + vp.logger.Info("secret not found", "secret", metal.AudittailerServerSecretName) } values := map[string]any{ From 028e01f197685802db0c7ff9e889eb4cbf5f7794 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 15:45:24 +0200 Subject: [PATCH 08/21] fix ccm-server-secret --- .../templates/cloud-controller-manager.yaml | 6 +++--- charts/internal/control-plane/values.yaml | 2 ++ pkg/controller/controlplane/valuesprovider.go | 11 ++++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/charts/internal/control-plane/templates/cloud-controller-manager.yaml b/charts/internal/control-plane/templates/cloud-controller-manager.yaml index d8e4b5aa2..ce7eb3ffb 100644 --- a/charts/internal/control-plane/templates/cloud-controller-manager.yaml +++ b/charts/internal/control-plane/templates/cloud-controller-manager.yaml @@ -68,8 +68,8 @@ spec: - --secure-port={{ include "cloud-controller-manager.port" . }} - --authentication-kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig - --authorization-kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig - - --tls-cert-file=/var/lib/cloud-controller-manager-server/cloud-controller-manager-server.crt - - --tls-private-key-file=/var/lib/cloud-controller-manager-server/cloud-controller-manager-server.key + - --tls-cert-file=/var/lib/cloud-controller-manager-server/tls.crt + - --tls-private-key-file=/var/lib/cloud-controller-manager-server/tls.key - --tls-cipher-suites={{ include "kubernetes.tlsCipherSuites" . | replace "\n" "," | trimPrefix "," }} - --use-service-account-credentials - --v=2 @@ -143,7 +143,7 @@ spec: optional: false - name: cloud-controller-manager-server secret: - secretName: cloud-controller-manager-server + secretName: {{ .Values.cloudControllerManager.secrets.server }} - name: cloudprovider secret: secretName: cloudprovider diff --git a/charts/internal/control-plane/values.yaml b/charts/internal/control-plane/values.yaml index 3e02e2bb9..0f4dbb062 100644 --- a/charts/internal/control-plane/values.yaml +++ b/charts/internal/control-plane/values.yaml @@ -42,6 +42,8 @@ cloudControllerManager: limits: cpu: 250m memory: 300Mi + secrets: + server: cloud-controller-manager-server accountingExporter: enabled: false diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 5560146d3..ceebd73db 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -481,7 +481,7 @@ func (vp *valuesProvider) GetControlPlaneChartValues( return nil, fmt.Errorf("could not retrieve project from metal-api %w", err) } - ccmValues, err := getCCMChartValues(ctx, cpConfig, infrastructureConfig, infrastructure, cluster, checksums, scaledDown, mclient, metalControlPlane, nws) + ccmValues, err := getCCMChartValues(ctx, cpConfig, infrastructureConfig, infrastructure, cluster, checksums, scaledDown, mclient, metalControlPlane, nws, secretsReader) if err != nil { return nil, err } @@ -1033,6 +1033,7 @@ func getCCMChartValues( mclient metalgo.Client, mcp *apismetal.MetalControlPlane, nws networkMap, + secretsReader secretsmanager.Reader, ) (map[string]interface{}, error) { projectID := infrastructureConfig.ProjectID nodeCIDR := infrastructure.Status.NodesCIDR @@ -1101,6 +1102,11 @@ func getCCMChartValues( } } + serverSecret, found := secretsReader.Get(metal.CloudControllerManagerServerName) + if !found { + return nil, fmt.Errorf("secret %q not found", metal.CloudControllerManagerServerName) + } + values := map[string]interface{}{ "kubernetesVersion": cluster.Shoot.Spec.Kubernetes.Version, "cloudControllerManager": map[string]interface{}{ @@ -1121,6 +1127,9 @@ func getCCMChartValues( "checksum/secret-cloudprovider": checksums[v1alpha1constants.SecretNameCloudProvider], "checksum/configmap-cloud-provider-config": checksums[metal.CloudProviderConfigName], }, + "secrets": map[string]any{ + "server": serverSecret.Name, + }, }, } From 2c11071340e4d0c5eb4527bc331d415592a5bdf0 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 15:46:06 +0200 Subject: [PATCH 09/21] try fixing tailer-secrets --- pkg/controller/controlplane/valuesprovider.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index ceebd73db..fc15f3232 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -684,13 +684,15 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, m }) } + droptailerServerSecretName := metal.DroptailerClientSecretName droptailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.DroptailerNamespace, metal.DroptailerServerSecretName) - if err != nil { - vp.logger.Info("secret not found", "secret", metal.DroptailerServerSecretName) + if err == nil { + droptailerServerSecretName = droptailerServerSecret.Name } + audittailerServerSecretName := metal.AudittailerClientSecretName audittailerServerSecret, err := vp.getSecretFromShoot(ctx, cluster, metal.AudittailerNamespace, metal.AudittailerServerSecretName) - if err != nil { - vp.logger.Info("secret not found", "secret", metal.AudittailerServerSecretName) + if err == nil { + audittailerServerSecretName = audittailerServerSecret.Name } values := map[string]any{ From 0c47e8050a65824b5f0e0661aa75f15662d28285 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 15:56:44 +0200 Subject: [PATCH 10/21] fix --- pkg/controller/controlplane/valuesprovider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index fc15f3232..398f4dbae 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -711,10 +711,10 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, m "destinations": egressDestinations, }, "droptailer": map[string]any{ - "secretName": droptailerServerSecret.Name, + "secretName": droptailerServerSecretName, }, "audittailer": map[string]any{ - "secretName": audittailerServerSecret.Name, + "secretName": audittailerServerSecretName, }, } From 01e2507d1f343dfebdf14fac83c5ee097078c105 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 15:57:01 +0200 Subject: [PATCH 11/21] fix fwcm secret --- charts/internal/control-plane/values.yaml | 2 ++ pkg/controller/controlplane/valuesprovider.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/internal/control-plane/values.yaml b/charts/internal/control-plane/values.yaml index 0f4dbb062..3685be4d9 100644 --- a/charts/internal/control-plane/values.yaml +++ b/charts/internal/control-plane/values.yaml @@ -21,6 +21,8 @@ firewallControllerManager: -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- + secrets: + server: firewall-controller-manager cloudControllerManager: additionalParameters: [] diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 398f4dbae..ceab3c23e 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -506,7 +506,7 @@ func (vp *valuesProvider) GetControlPlaneChartValues( return nil, fmt.Errorf("could not find current ssh secret: %w", err) } - firewallValues, err := vp.getFirewallControllerManagerChartValues(ctx, cluster, metalControlPlane, sshSecret, caBundle) + firewallValues, err := vp.getFirewallControllerManagerChartValues(ctx, cluster, metalControlPlane, sshSecret, caBundle, secretsReader) if err != nil { return nil, err } @@ -1333,7 +1333,7 @@ func getStorageControlPlaneChartValues(ctx context.Context, client client.Client return values, nil } -func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Context, cluster *extensionscontroller.Cluster, metalControlPlane *apismetal.MetalControlPlane, sshSecret, caBundle *corev1.Secret) (map[string]any, error) { +func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Context, cluster *extensionscontroller.Cluster, metalControlPlane *apismetal.MetalControlPlane, sshSecret, caBundle *corev1.Secret, secretsReader secretsmanager.Reader) (map[string]any, error) { if cluster.Shoot.Spec.DNS.Domain == nil { return nil, fmt.Errorf("cluster dns domain is not yet set") } @@ -1366,6 +1366,10 @@ func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Co if err != nil && !apierrors.IsNotFound(err) { return nil, err } + serverSecret, found := secretsReader.Get(metal.FirewallControllerManagerDeploymentName) + if !found { + return nil, fmt.Errorf("secret %q not found", metal.CloudControllerManagerServerName) + } return map[string]any{ "firewallControllerManager": map[string]any{ @@ -1382,6 +1386,9 @@ func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Co "url": metalControlPlane.Endpoint, }, "caBundle": strings.TrimSpace(string(caBundle.Data["bundle.crt"])), + "secrets": map[string]any{ + "server": serverSecret.Name, + }, }, }, nil } From 2e78d73bedbe727d9036d8089bb766016a7128c9 Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 16:07:04 +0200 Subject: [PATCH 12/21] fix --- .../templates/firewall-controller-manager.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/charts/internal/control-plane/templates/firewall-controller-manager.yaml b/charts/internal/control-plane/templates/firewall-controller-manager.yaml index 95c4c41d6..19b3bc005 100644 --- a/charts/internal/control-plane/templates/firewall-controller-manager.yaml +++ b/charts/internal/control-plane/templates/firewall-controller-manager.yaml @@ -156,12 +156,7 @@ spec: volumes: - name: webhook-certs secret: - secretName: firewall-controller-manager - items: - - key: firewall-controller-manager.crt - path: tls.crt - - key: firewall-controller-manager.key - path: tls.key + secretName: {{ .Values.firewallControllerManager.secrets.server }} - name: token-dir emptyDir: {} --- From bde21c472c31b02f796474d10deb48483f58cf5d Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 16:18:57 +0200 Subject: [PATCH 13/21] fix mutatingwebhook --- pkg/apis/metal/helper/secrets.go | 6 +++--- pkg/controller/controlplane/valuesprovider.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/metal/helper/secrets.go b/pkg/apis/metal/helper/secrets.go index 532092a7f..84074300b 100644 --- a/pkg/apis/metal/helper/secrets.go +++ b/pkg/apis/metal/helper/secrets.go @@ -11,6 +11,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal" ) func GetLatestSSHSecret(ctx context.Context, c client.Client, namespace string) (*corev1.Secret, error) { @@ -29,9 +30,8 @@ func GetLatestSSHSecret(ctx context.Context, c client.Client, namespace string) func GetLatestCABundle(ctx context.Context, c client.Client, namespace string) (*corev1.Secret, error) { secretList := &corev1.SecretList{} if err := c.List(ctx, secretList, client.InNamespace(namespace), client.MatchingLabels{ - secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, - secretsmanager.LabelKeyManagerIdentity: constants.SecretManagerIdentityGardenlet, - secretsmanager.LabelKeyName: "ca-bundle", + secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, + secretsmanager.LabelKeyName: metal.FirewallControllerManagerDeploymentName, }); err != nil { return nil, err } diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index ceab3c23e..5427a5ae2 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -97,7 +97,7 @@ func secretConfigsFunc(namespace string) []extensionssecretsmanager.SecretConfig CommonName: metal.FirewallControllerManagerDeploymentName, DNSNames: kutil.DNSNamesForService(metal.FirewallControllerManagerDeploymentName, namespace), CertType: secrets.ServerCert, - SkipPublishingCACertificate: true, + SkipPublishingCACertificate: false, }, // use current CA for signing server cert to prevent mismatches when dropping the old CA from the webhook // config in phase Completing @@ -1385,7 +1385,7 @@ func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Co "metalapi": map[string]any{ "url": metalControlPlane.Endpoint, }, - "caBundle": strings.TrimSpace(string(caBundle.Data["bundle.crt"])), + "caBundle": strings.TrimSpace(string(caBundle.Data["ca.crt"])), "secrets": map[string]any{ "server": serverSecret.Name, }, From 914d6c2bae443880afabd8c962016a3b1ebfd80d Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Fri, 21 Apr 2023 16:38:40 +0200 Subject: [PATCH 14/21] remove unused mount --- pkg/webhook/controlplane/ensurer.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index 63b1d745b..c552b18f8 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -136,14 +136,6 @@ var ( }, }, } - audittailerClientSecretVolume = corev1.Volume{ - Name: metal.AudittailerClientSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: metal.AudittailerClientSecretName, - }, - }, - } auditForwarderSplunkConfigVolumeMount = corev1.VolumeMount{ Name: metal.AuditForwarderSplunkConfigName, MountPath: "/fluent-bit/etc/add", @@ -311,11 +303,6 @@ var ( }, }, VolumeMounts: []corev1.VolumeMount{ - { - Name: audittailerClientSecretVolume.Name, - ReadOnly: true, - MountPath: "/shootconfig", - }, { Name: "kubeconfig", MountPath: gutil.VolumeMountPathGenericKubeconfig, @@ -338,7 +325,6 @@ func ensureVolumes(ps *corev1.PodSpec, makeAuditForwarder, auditToSplunk bool) { ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, auditKubeconfig) ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, auditPolicyVolume) ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, auditLogVolume) - ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, audittailerClientSecretVolume) } if auditToSplunk { ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, auditForwarderSplunkConfigVolume) From 3f6b18af7e75dfec34ec7c6dff71d4998b5476bc Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Mon, 24 Apr 2023 12:54:56 +0200 Subject: [PATCH 15/21] do not enable auditforwarder unless token is set --- pkg/webhook/controlplane/ensurer.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index c552b18f8..878d96c80 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -82,6 +82,17 @@ func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, gctx gconte if validation.ClusterAuditEnabled(&e.controllerConfig, cpConfig) { makeAuditForwarder = true } + if makeAuditForwarder { + audittailersecret := &corev1.Secret{} + if err := e.client.Get(ctx, kutil.Key(cluster.ObjectMeta.Name, "shoot-access-audittailer-client"), audittailersecret); err != nil { + logger.Error(err, "could not get shoot-access-audittailer-client for cluster", "cluster name", cluster.ObjectMeta.Name) + makeAuditForwarder = false + } + if len(audittailersecret.Data) == 0 { + logger.Error(err, "token for shoot-access-audittailer-client not yet set in cluster", "cluster name", cluster.ObjectMeta.Name) + makeAuditForwarder = false + } + } auditToSplunk := false if validation.AuditToSplunkEnabled(&e.controllerConfig, cpConfig) { From d4ff73d62076d75b376ff19aa81abaa6f1ceb2b0 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 24 Apr 2023 14:34:45 +0200 Subject: [PATCH 16/21] Review current PR --- pkg/apis/metal/helper/secrets.go | 61 ++----------------- pkg/controller/controlplane/valuesprovider.go | 28 +++++---- pkg/metal/types.go | 2 + pkg/secret/secret.go | 50 +++++++++++++++ pkg/webhook/controlplane/ensurer.go | 8 +-- 5 files changed, 77 insertions(+), 72 deletions(-) create mode 100644 pkg/secret/secret.go diff --git a/pkg/apis/metal/helper/secrets.go b/pkg/apis/metal/helper/secrets.go index 84074300b..fab3bd83d 100644 --- a/pkg/apis/metal/helper/secrets.go +++ b/pkg/apis/metal/helper/secrets.go @@ -2,9 +2,6 @@ package helper import ( "context" - "fmt" - "strconv" - "time" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -12,6 +9,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal" + "github.com/metal-stack/gardener-extension-provider-metal/pkg/secret" ) func GetLatestSSHSecret(ctx context.Context, c client.Client, namespace string) (*corev1.Secret, error) { @@ -24,65 +22,18 @@ func GetLatestSSHSecret(ctx context.Context, c client.Client, namespace string) return nil, err } - return getLatestIssuedSecret(secretList.Items) -} - -func GetLatestCABundle(ctx context.Context, c client.Client, namespace string) (*corev1.Secret, error) { - secretList := &corev1.SecretList{} - if err := c.List(ctx, secretList, client.InNamespace(namespace), client.MatchingLabels{ - secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, - secretsmanager.LabelKeyName: metal.FirewallControllerManagerDeploymentName, - }); err != nil { - return nil, err - } - - return getLatestIssuedSecret(secretList.Items) + return secret.GetLatestIssuedSecret(secretList.Items) } func GetLatestSecret(ctx context.Context, c client.Client, namespace string, name string) (*corev1.Secret, error) { secretList := &corev1.SecretList{} if err := c.List(ctx, secretList, client.InNamespace(namespace), client.MatchingLabels{ - secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, - secretsmanager.LabelKeyName: name, + secretsmanager.LabelKeyManagedBy: secretsmanager.LabelValueSecretsManager, + secretsmanager.LabelKeyManagerIdentity: metal.ManagerIdentity, + secretsmanager.LabelKeyName: name, }); err != nil { return nil, err } - return getLatestIssuedSecret(secretList.Items) -} - -// getLatestIssuedSecret returns the secret with the "issued-at-time" label that represents the latest point in time -func getLatestIssuedSecret(secrets []corev1.Secret) (*corev1.Secret, error) { - if len(secrets) == 0 { - return nil, fmt.Errorf("no secret found") - } - - var newestSecret *corev1.Secret - var currentIssuedAtTime time.Time - for i := 0; i < len(secrets); i++ { - // if some of the secrets have no "issued-at-time" label - // we have a problem since this is the source of truth - issuedAt, ok := secrets[i].Labels[secretsmanager.LabelKeyIssuedAtTime] - if !ok { - // there are some old secrets from ancient gardener versions which have to be skipped... (e.g. ssh-keypair.old) - continue - } - - issuedAtUnix, err := strconv.ParseInt(issuedAt, 10, 64) - if err != nil { - return nil, err - } - - issuedAtTime := time.Unix(issuedAtUnix, 0).UTC() - if newestSecret == nil || issuedAtTime.After(currentIssuedAtTime) { - newestSecret = &secrets[i] - currentIssuedAtTime = issuedAtTime - } - } - - if newestSecret == nil { - return nil, fmt.Errorf("no secret found") - } - - return newestSecret, nil + return secret.GetLatestIssuedSecret(secretList.Items) } diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 5427a5ae2..5ef0a5871 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -89,7 +89,7 @@ func secretConfigsFunc(namespace string) []extensionssecretsmanager.SecretConfig CertType: secrets.ServerCert, SkipPublishingCACertificate: true, }, - Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(v1alpha1constants.SecretNameCACluster)}, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(v1alpha1constants.SecretNameCACluster, secretsmanager.UseCurrentCA)}, }, { Config: &secrets.CertificateSecretConfig{ @@ -501,7 +501,7 @@ func (vp *valuesProvider) GetControlPlaneChartValues( return nil, fmt.Errorf("could not find current ssh secret: %w", err) } - caBundle, err := helper.GetLatestCABundle(ctx, vp.Client(), cp.Namespace) + caBundle, err := helper.GetLatestSecret(ctx, vp.Client(), cp.Namespace, metal.FirewallControllerManagerDeploymentName) if err != nil { return nil, fmt.Errorf("could not find current ssh secret: %w", err) } @@ -594,7 +594,6 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues(ctx context.Context, c return nil, err } - // FIXME stefan what to do here if !extensionscontroller.IsHibernated(cluster) { if err := vp.deploySecretsToShoot(ctx, cluster, metal.AudittailerNamespace, vp.audittailerSecretConfigs); err != nil { vp.logger.Error(err, "error deploying audittailer certs") @@ -872,11 +871,12 @@ func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager. return nil } + const auditTailerCAName = "ca-provider-metal-audittailer" return []extensionssecretsmanager.SecretConfigWithOptions{ { Config: &secrets.CertificateSecretConfig{ - Name: "ca-provider-metal-audittailer", - CommonName: "ca-provider-metal-audittailer", + Name: auditTailerCAName, + CommonName: auditTailerCAName, CertType: secrets.CACert, }, Options: []secretsmanager.GenerateOption{secretsmanager.Persist()}, @@ -890,7 +890,7 @@ func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager. CertType: secrets.ClientCert, SkipPublishingCACertificate: false, }, - Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(auditTailerCAName, secretsmanager.UseCurrentCA)}, }, { Config: &secrets.CertificateSecretConfig{ @@ -901,17 +901,19 @@ func (vp *valuesProvider) audittailerSecretConfigs() []extensionssecretsmanager. CertType: secrets.ServerCert, SkipPublishingCACertificate: false, }, - Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-audittailer")}, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(auditTailerCAName, secretsmanager.UseCurrentCA)}, }, } } func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.SecretConfigWithOptions { + + const droptailerCAName = "ca-provider-metal-droptailer" return []extensionssecretsmanager.SecretConfigWithOptions{ { Config: &secrets.CertificateSecretConfig{ - Name: "ca-provider-metal-droptailer", - CommonName: "ca-provider-metal-droptailer", + Name: droptailerCAName, + CommonName: droptailerCAName, CertType: secrets.CACert, }, Options: []secretsmanager.GenerateOption{secretsmanager.Persist()}, @@ -925,7 +927,7 @@ func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.S CertType: secrets.ClientCert, SkipPublishingCACertificate: false, }, - Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(droptailerCAName, secretsmanager.UseCurrentCA)}, }, { Config: &secrets.CertificateSecretConfig{ @@ -936,7 +938,7 @@ func (vp *valuesProvider) droptailerSecretConfigs() []extensionssecretsmanager.S CertType: secrets.ServerCert, SkipPublishingCACertificate: false, }, - Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA("ca-provider-metal-droptailer")}, + Options: []secretsmanager.GenerateOption{secretsmanager.SignedByCA(droptailerCAName, secretsmanager.UseCurrentCA)}, }, } } @@ -964,7 +966,7 @@ func (vp *valuesProvider) deploySecretsToShoot(ctx context.Context, cluster *ext return fmt.Errorf("could not ensure namespace: %w", err) } - manager, err := secretsmanager.New(ctx, vp.logger.WithName("shoot-secrets-manager"), clock.RealClock{}, c, namespace, metal.Type+"-provider-shoot-controlplane", nil) + manager, err := secretsmanager.New(ctx, vp.logger.WithName("shoot-secrets-manager"), clock.RealClock{}, c, namespace, metal.ManagerIdentity, nil) if err != nil { return fmt.Errorf("unable to create secrets manager: %w", err) } @@ -1368,7 +1370,7 @@ func (vp *valuesProvider) getFirewallControllerManagerChartValues(ctx context.Co } serverSecret, found := secretsReader.Get(metal.FirewallControllerManagerDeploymentName) if !found { - return nil, fmt.Errorf("secret %q not found", metal.CloudControllerManagerServerName) + return nil, fmt.Errorf("secret %q not found", metal.FirewallControllerManagerDeploymentName) } return map[string]any{ diff --git a/pkg/metal/types.go b/pkg/metal/types.go index 64eb67f80..8d76a0c7b 100644 --- a/pkg/metal/types.go +++ b/pkg/metal/types.go @@ -77,6 +77,8 @@ const ( FirewallControllerManagerDeploymentName = "firewall-controller-manager" // FirewallDeploymentName is the name of the firewall deployment deployed to the seed cluster to get managed by the FCM. FirewallDeploymentName = "shoot-firewall" + // ManagerIdentity is put as a label to every secret managed by the gepm and secretsmanager to make searching easier + ManagerIdentity = Type + "-provider-shoot-controlplane" ) var ( diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go new file mode 100644 index 000000000..37ddd3293 --- /dev/null +++ b/pkg/secret/secret.go @@ -0,0 +1,50 @@ +// Package secret is extracted to be reused by other metal components which require to gather the latest secret. +// which was created by the gardener secretsmanager. +// We dont want to include the whole gardener dependency for this sole purpose. +package secret + +import ( + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" +) + +const labelKeyIssuedAtTime = "issued-at-time" + +// getLatestIssuedSecret returns the secret with the "issued-at-time" label that represents the latest point in time +func GetLatestIssuedSecret(secrets []corev1.Secret) (*corev1.Secret, error) { + if len(secrets) == 0 { + return nil, fmt.Errorf("no secret found") + } + + var newestSecret *corev1.Secret + var currentIssuedAtTime time.Time + for i := 0; i < len(secrets); i++ { + // if some of the secrets have no "issued-at-time" label + // we have a problem since this is the source of truth + issuedAt, ok := secrets[i].Labels[labelKeyIssuedAtTime] + if !ok { + // there are some old secrets from ancient gardener versions which have to be skipped... (e.g. ssh-keypair.old) + continue + } + + issuedAtUnix, err := strconv.ParseInt(issuedAt, 10, 64) + if err != nil { + return nil, err + } + + issuedAtTime := time.Unix(issuedAtUnix, 0).UTC() + if newestSecret == nil || issuedAtTime.After(currentIssuedAtTime) { + newestSecret = &secrets[i] + currentIssuedAtTime = issuedAtTime + } + } + + if newestSecret == nil { + return nil, fmt.Errorf("no secret found") + } + + return newestSecret, nil +} diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index 878d96c80..3d6ec0967 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -84,12 +84,12 @@ func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, gctx gconte } if makeAuditForwarder { audittailersecret := &corev1.Secret{} - if err := e.client.Get(ctx, kutil.Key(cluster.ObjectMeta.Name, "shoot-access-audittailer-client"), audittailersecret); err != nil { - logger.Error(err, "could not get shoot-access-audittailer-client for cluster", "cluster name", cluster.ObjectMeta.Name) + if err := e.client.Get(ctx, kutil.Key(cluster.ObjectMeta.Name, gutil.SecretNamePrefixShootAccess+metal.AudittailerClientSecretName), audittailersecret); err != nil { + logger.Error(err, "could not get secret for cluster", "secret", gutil.SecretNamePrefixShootAccess+metal.AudittailerClientSecretName, "cluster name", cluster.ObjectMeta.Name) makeAuditForwarder = false } if len(audittailersecret.Data) == 0 { - logger.Error(err, "token for shoot-access-audittailer-client not yet set in cluster", "cluster name", cluster.ObjectMeta.Name) + logger.Error(err, "token for secret not yet set in cluster", "secret", gutil.SecretNamePrefixShootAccess+metal.AudittailerClientSecretName, "cluster name", cluster.ObjectMeta.Name) makeAuditForwarder = false } } @@ -231,7 +231,7 @@ var ( }, Optional: pointer.Pointer(false), LocalObjectReference: corev1.LocalObjectReference{ - Name: "shoot-access-" + metal.AudittailerClientSecretName, + Name: gutil.SecretNamePrefixShootAccess + metal.AudittailerClientSecretName, }, }, }, From 1a3e45d0a686032ad8145270d4d4c53068f6d95f Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Mon, 24 Apr 2023 17:07:45 +0200 Subject: [PATCH 17/21] fix error message --- pkg/controller/controlplane/valuesprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 5ef0a5871..6c302a913 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -503,7 +503,7 @@ func (vp *valuesProvider) GetControlPlaneChartValues( caBundle, err := helper.GetLatestSecret(ctx, vp.Client(), cp.Namespace, metal.FirewallControllerManagerDeploymentName) if err != nil { - return nil, fmt.Errorf("could not find current ssh secret: %w", err) + return nil, fmt.Errorf("could not get ca from secret: %w", err) } firewallValues, err := vp.getFirewallControllerManagerChartValues(ctx, cluster, metalControlPlane, sshSecret, caBundle, secretsReader) From 16f6300787f71b4150fc2df98709e7f28141974b Mon Sep 17 00:00:00 2001 From: Markus Wennrich Date: Mon, 24 Apr 2023 17:08:43 +0200 Subject: [PATCH 18/21] try fixing secretsmanager identity --- pkg/metal/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metal/types.go b/pkg/metal/types.go index 8d76a0c7b..bc42c3fd3 100644 --- a/pkg/metal/types.go +++ b/pkg/metal/types.go @@ -78,7 +78,7 @@ const ( // FirewallDeploymentName is the name of the firewall deployment deployed to the seed cluster to get managed by the FCM. FirewallDeploymentName = "shoot-firewall" // ManagerIdentity is put as a label to every secret managed by the gepm and secretsmanager to make searching easier - ManagerIdentity = Type + "-provider-shoot-controlplane" + ManagerIdentity = "provider-" + Type + "-controlplane" ) var ( From 0ef326f85afcaa9b78dfa1d75235ca46154777ee Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 25 Apr 2023 09:19:01 +0200 Subject: [PATCH 19/21] Add missing RBAC for duros-controller because he needs to grant these also to the csi-driver --- charts/internal/shoot-control-plane/templates/rbac-duros.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/internal/shoot-control-plane/templates/rbac-duros.yaml b/charts/internal/shoot-control-plane/templates/rbac-duros.yaml index c014a88fd..3d39c0788 100644 --- a/charts/internal/shoot-control-plane/templates/rbac-duros.yaml +++ b/charts/internal/shoot-control-plane/templates/rbac-duros.yaml @@ -11,11 +11,13 @@ rules: - volumesnapshotclasses - volumesnapshotcontents - volumesnapshots + - volumesnapshots/status verbs: - create - delete - get - list + - watch - patch - update - apiGroups: @@ -36,6 +38,7 @@ rules: - csidrivers - csinodes - volumeattachments + - volumeattachments/status - storageclasses verbs: - create From babe73055221a566974ac7f601b57596b36e2cb6 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 25 Apr 2023 09:43:41 +0200 Subject: [PATCH 20/21] Add missing RBAC --- charts/internal/shoot-control-plane/templates/rbac-duros.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/internal/shoot-control-plane/templates/rbac-duros.yaml b/charts/internal/shoot-control-plane/templates/rbac-duros.yaml index 3d39c0788..5bb5d025e 100644 --- a/charts/internal/shoot-control-plane/templates/rbac-duros.yaml +++ b/charts/internal/shoot-control-plane/templates/rbac-duros.yaml @@ -10,6 +10,7 @@ rules: resources: - volumesnapshotclasses - volumesnapshotcontents + - volumesnapshotcontents/status - volumesnapshots - volumesnapshots/status verbs: From 25ea508c5eb474fc966f8ad0b32b930d4386b06e Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 25 Apr 2023 13:30:35 +0200 Subject: [PATCH 21/21] Update fluentd from 1.12 -> 1.16 which is 2 years of development --- charts/images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/images.yaml b/charts/images.yaml index 5a44d5347..7d33192b8 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -18,7 +18,7 @@ images: - name: audittailer sourceRepository: https://github.com/fluent/fluentd repository: fluent/fluentd - tag: "v1.12" + tag: "v1.16" - name: droptailer sourceRepository: github.com/metal-stack/droptailer repository: ghcr.io/metal-stack/droptailer