diff --git a/controllers/aso_credential_cache.go b/controllers/aso_credential_cache.go new file mode 100644 index 00000000000..2a84334044c --- /dev/null +++ b/controllers/aso_credential_cache.go @@ -0,0 +1,221 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "os" + "strconv" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel" + asoannotations "github.com/Azure/azure-service-operator/v2/pkg/common/annotations" + "github.com/Azure/azure-service-operator/v2/pkg/common/config" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/pkg/ot" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +const ( + asoNamespaceSecretName = "aso-credential" //nolint:gosec // This is not a secret, only a reference to one. + asoGlobalSecretName = "aso-controller-settings" + asoNamespaceAnnotation = "serviceoperator.azure.com/operator-namespace" +) + +// ASOCredentialCache caches credentials defined for ASO resources. +type ASOCredentialCache interface { + authTokenForASOResource(context.Context, client.Object) (azcore.TokenCredential, error) +} + +type asoCredentialCache struct { + cache azure.CredentialCache + client client.Client +} + +// NewASOCredentialCache creates a new ASOCredentialCache. +func NewASOCredentialCache(cache azure.CredentialCache, client client.Client) ASOCredentialCache { + return &asoCredentialCache{ + cache: cache, + client: client, + } +} + +func (c *asoCredentialCache) authTokenForASOResource(ctx context.Context, obj client.Object) (azcore.TokenCredential, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.asoCredentialCache.authTokenForASOResource") + defer done() + + clientOpts, err := c.clientOptsForASOResource(ctx, obj) + if err != nil { + return nil, err + } + + secretName := asoNamespaceSecretName + if resourceSecretName := obj.GetAnnotations()[asoannotations.PerResourceSecret]; resourceSecretName != "" { + secretName = resourceSecretName + } + secret := &corev1.Secret{} + err = c.client.Get(ctx, client.ObjectKey{Namespace: obj.GetNamespace(), Name: secretName}, secret) + if client.IgnoreNotFound(err) != nil { + return nil, err + } + if err == nil { + return c.authTokenForScopedASOSecret(secret, clientOpts) + } + + secretNamespace := obj.GetAnnotations()[asoNamespaceAnnotation] + err = c.client.Get(ctx, client.ObjectKey{Namespace: secretNamespace, Name: asoGlobalSecretName}, secret) + if err != nil { + return nil, err + } + + return c.authTokenForGlobalASOSecret(secret, clientOpts) +} + +func (c *asoCredentialCache) clientOptsForASOResource(ctx context.Context, obj client.Object) (azcore.ClientOptions, error) { + secretNamespace := obj.GetAnnotations()[asoNamespaceAnnotation] + secret := &corev1.Secret{} + err := c.client.Get(ctx, client.ObjectKey{Namespace: secretNamespace, Name: asoGlobalSecretName}, secret) + if client.IgnoreNotFound(err) != nil { + return azcore.ClientOptions{}, err + } + + otelTP, err := ot.OTLPTracerProvider(ctx) + if err != nil { + return azcore.ClientOptions{}, err + } + + opts := azcore.ClientOptions{ + TracingProvider: azotel.NewTracingProvider(otelTP, nil), + Cloud: cloud.Configuration{ + ActiveDirectoryAuthorityHost: string(secret.Data[config.AzureAuthorityHost]), + }, + } + + if len(secret.Data[config.ResourceManagerAudience]) > 0 || + len(secret.Data[config.ResourceManagerEndpoint]) > 0 { + opts.Cloud.Services = map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: { + Audience: string(secret.Data[config.ResourceManagerAudience]), + Endpoint: string(secret.Data[config.ResourceManagerEndpoint]), + }, + } + } + + return opts, nil +} + +func (c *asoCredentialCache) authTokenForScopedASOSecret(secret *corev1.Secret, clientOpts azcore.ClientOptions) (azcore.TokenCredential, error) { + d := secret.Data + + if _, hasSecret := d[config.AzureClientSecret]; hasSecret { + return c.cache.GetOrStoreClientSecret( + string(d[config.AzureTenantID]), + string(d[config.AzureClientID]), + string(d[config.AzureClientSecret]), + &azidentity.ClientSecretCredentialOptions{ + ClientOptions: clientOpts, + }, + ) + } + + if _, hasCert := d[config.AzureClientCertificate]; hasCert { + return c.cache.GetOrStoreClientCert( + string(d[config.AzureTenantID]), + string(d[config.AzureClientID]), + d[config.AzureClientCertificate], + d[config.AzureClientCertificatePassword], + &azidentity.ClientCertificateCredentialOptions{ + ClientOptions: clientOpts, + }, + ) + } + + if authMode := d[config.AuthMode]; config.AuthModeOption(authMode) == config.PodIdentityAuthMode { + return c.cache.GetOrStoreManagedIdentity( + &azidentity.ManagedIdentityCredentialOptions{ + ClientOptions: clientOpts, + ID: azidentity.ClientID(d[config.AzureClientID]), + }, + ) + } + + return c.cache.GetOrStoreWorkloadIdentity( + &azidentity.WorkloadIdentityCredentialOptions{ + ClientOptions: clientOpts, + TenantID: string(d[config.AzureTenantID]), + ClientID: string(d[config.AzureClientID]), + TokenFilePath: federatedTokenFilePath(), + }, + ) +} + +func (c *asoCredentialCache) authTokenForGlobalASOSecret(secret *corev1.Secret, clientOpts azcore.ClientOptions) (azcore.TokenCredential, error) { + d := secret.Data + + if workloadID, _ := strconv.ParseBool(string(d[config.UseWorkloadIdentityAuth])); workloadID { + return c.cache.GetOrStoreWorkloadIdentity( + &azidentity.WorkloadIdentityCredentialOptions{ + ClientOptions: clientOpts, + TenantID: string(d[config.AzureTenantID]), + ClientID: string(d[config.AzureClientID]), + TokenFilePath: federatedTokenFilePath(), + }, + ) + } + + if _, hasSecret := d[config.AzureClientSecret]; hasSecret { + return c.cache.GetOrStoreClientSecret( + string(d[config.AzureTenantID]), + string(d[config.AzureClientID]), + string(d[config.AzureClientSecret]), + &azidentity.ClientSecretCredentialOptions{ + ClientOptions: clientOpts, + }, + ) + } + + if _, hasCert := d[config.AzureClientCertificate]; hasCert { + return c.cache.GetOrStoreClientCert( + string(d[config.AzureTenantID]), + string(d[config.AzureClientID]), + d[config.AzureClientCertificate], + d[config.AzureClientCertificatePassword], + &azidentity.ClientCertificateCredentialOptions{ + ClientOptions: clientOpts, + }, + ) + } + + return c.cache.GetOrStoreManagedIdentity( + &azidentity.ManagedIdentityCredentialOptions{ + ClientOptions: clientOpts, + ID: azidentity.ClientID(d[config.AzureClientID]), + }, + ) +} + +func federatedTokenFilePath() string { + if env, ok := os.LookupEnv("AZURE_FEDERATED_TOKEN_FILE"); ok { + return env + } + return "/var/run/secrets/azure/tokens/azure-identity-token" +} diff --git a/controllers/aso_credential_cache_test.go b/controllers/aso_credential_cache_test.go new file mode 100644 index 00000000000..3b7943d6fab --- /dev/null +++ b/controllers/aso_credential_cache_test.go @@ -0,0 +1,488 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "reflect" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" + asoannotations "github.com/Azure/azure-service-operator/v2/pkg/common/annotations" + "github.com/Azure/azure-service-operator/v2/pkg/common/config" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/mock_azure" +) + +type credentialParams struct { + credentialType azure.CredentialType + tenantID string + clientID string + clientSecret string + clientCert []byte + clientCertPassword []byte + + authorityHost string + armEndpoint string + armAudience string +} + +func TestAuthTokenForASOResource(t *testing.T) { + tests := []struct { + name string + resource client.Object + secret *corev1.Secret + expectedParams credentialParams + expectedErr error + }{ + { + name: "per-resource secret client secret", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoannotations.PerResourceSecret: "my-secret", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientSecret: []byte("hunter2"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientSecret, + tenantID: "tenant", + clientID: "client", + clientSecret: "hunter2", + }, + }, + { + name: "per-resource secret client cert", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoannotations.PerResourceSecret: "my-secret", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientCertificate: []byte("cert"), + config.AzureClientCertificatePassword: []byte("hunter2"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientCert, + tenantID: "tenant", + clientID: "client", + clientCert: []byte("cert"), + clientCertPassword: []byte("hunter2"), + }, + }, + { + name: "per-resource secret managed identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoannotations.PerResourceSecret: "my-secret", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureClientID: []byte("client"), + config.AuthMode: []byte(config.PodIdentityAuthMode), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeManagedIdentity, + clientID: "client", + }, + }, + { + name: "per-resource secret workload identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoannotations.PerResourceSecret: "my-secret", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeWorkloadIdentity, + tenantID: "tenant", + clientID: "client", + }, + }, + { + name: "namespace secret client secret", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoNamespaceSecretName, + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientSecret: []byte("hunter2"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientSecret, + tenantID: "tenant", + clientID: "client", + clientSecret: "hunter2", + }, + }, + { + name: "namespace secret client cert", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoNamespaceSecretName, + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientCertificate: []byte("cert"), + config.AzureClientCertificatePassword: []byte("hunter2"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientCert, + tenantID: "tenant", + clientID: "client", + clientCert: []byte("cert"), + clientCertPassword: []byte("hunter2"), + }, + }, + { + name: "namespace secret managed identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoNamespaceSecretName, + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureClientID: []byte("client"), + config.AuthMode: []byte(config.PodIdentityAuthMode), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeManagedIdentity, + clientID: "client", + }, + }, + { + name: "namespace secret workload identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoNamespaceSecretName, + Namespace: "namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeWorkloadIdentity, + tenantID: "tenant", + clientID: "client", + }, + }, + { + name: "global secret client secret", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoNamespaceAnnotation: "aso-namespace", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoGlobalSecretName, + Namespace: "aso-namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientSecret: []byte("hunter2"), + config.AzureAuthorityHost: []byte("auth host"), + config.ResourceManagerEndpoint: []byte("arm endpoint"), + config.ResourceManagerAudience: []byte("arm audience"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientSecret, + authorityHost: "auth host", + armEndpoint: "arm endpoint", + armAudience: "arm audience", + tenantID: "tenant", + clientID: "client", + clientSecret: "hunter2", + }, + }, + { + name: "global secret client cert", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoNamespaceAnnotation: "aso-namespace", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoGlobalSecretName, + Namespace: "aso-namespace", + }, + Data: map[string][]byte{ + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + config.AzureClientCertificate: []byte("cert"), + config.AzureClientCertificatePassword: []byte("hunter2"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeClientCert, + tenantID: "tenant", + clientID: "client", + clientCert: []byte("cert"), + clientCertPassword: []byte("hunter2"), + }, + }, + { + name: "global secret managed identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoNamespaceAnnotation: "aso-namespace", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoGlobalSecretName, + Namespace: "aso-namespace", + }, + Data: map[string][]byte{ + config.AzureClientID: []byte("client"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeManagedIdentity, + clientID: "client", + }, + }, + { + name: "global secret workload identity", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoNamespaceAnnotation: "aso-namespace", + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: asoGlobalSecretName, + Namespace: "aso-namespace", + }, + Data: map[string][]byte{ + config.UseWorkloadIdentityAuth: []byte("true"), + config.AzureTenantID: []byte("tenant"), + config.AzureClientID: []byte("client"), + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialTypeWorkloadIdentity, + tenantID: "tenant", + clientID: "client", + }, + }, + { + name: "secret not found", + resource: &asoresourcesv1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Annotations: map[string]string{ + asoannotations.PerResourceSecret: "my-secret", + }, + }, + }, + expectedParams: credentialParams{ + credentialType: azure.CredentialType(-1), // don't expect any calls to the cache + }, + secret: nil, + expectedErr: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "secrets"}, asoGlobalSecretName), // When the per-resource secret isn't found, we try to get the global one + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewGomegaWithT(t) + mockCtrl := gomock.NewController(t) + + var objs []client.Object + if test.secret != nil { + objs = append(objs, test.secret) + } + + c := fakeclient.NewClientBuilder(). + WithObjects(objs...). + Build() + + credCache := mock_azure.NewMockCredentialCache(mockCtrl) + + expectedClientOpts := azcore.ClientOptions{ + Cloud: cloud.Configuration{ + ActiveDirectoryAuthorityHost: test.expectedParams.authorityHost, + }, + } + if test.expectedParams.armAudience != "" || + test.expectedParams.armEndpoint != "" { + expectedClientOpts.Cloud.Services = map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: { + Audience: test.expectedParams.armAudience, + Endpoint: test.expectedParams.armEndpoint, + }, + } + } + + switch test.expectedParams.credentialType { + case azure.CredentialTypeClientSecret: + credCache.EXPECT().GetOrStoreClientSecret( + test.expectedParams.tenantID, + test.expectedParams.clientID, + test.expectedParams.clientSecret, + gomock.Cond(func(opts *azidentity.ClientSecretCredentialOptions) bool { + // ignore tracing provider + return reflect.DeepEqual(expectedClientOpts.Cloud, opts.Cloud) + }), + ).Return(nil, nil) + case azure.CredentialTypeClientCert: + credCache.EXPECT().GetOrStoreClientCert( + test.expectedParams.tenantID, + test.expectedParams.clientID, + test.expectedParams.clientCert, + test.expectedParams.clientCertPassword, + gomock.Cond(func(opts *azidentity.ClientCertificateCredentialOptions) bool { + // ignore tracing provider + return reflect.DeepEqual(expectedClientOpts.Cloud, opts.Cloud) + }), + ).Return(nil, nil) + case azure.CredentialTypeManagedIdentity: + credCache.EXPECT().GetOrStoreManagedIdentity( + gomock.Cond(func(opts *azidentity.ManagedIdentityCredentialOptions) bool { + // ignore tracing provider + return reflect.DeepEqual(expectedClientOpts.Cloud, opts.Cloud) && + reflect.DeepEqual(azidentity.ClientID(test.expectedParams.clientID), opts.ID) + }), + ).Return(nil, nil) + case azure.CredentialTypeWorkloadIdentity: + credCache.EXPECT().GetOrStoreWorkloadIdentity( + gomock.Cond(func(opts *azidentity.WorkloadIdentityCredentialOptions) bool { + // ignore tracing provider + return reflect.DeepEqual(expectedClientOpts.Cloud, opts.Cloud) && + opts.TenantID == test.expectedParams.tenantID && + opts.ClientID == test.expectedParams.clientID + // ignore token file path, it's always the same + }), + ).Return(nil, nil) + } + + asoCache := &asoCredentialCache{ + cache: credCache, + client: c, + } + _, err := asoCache.authTokenForASOResource(context.Background(), test.resource) + if test.expectedErr != nil { + g.Expect(err).To(MatchError(test.expectedErr)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/controllers/azureasomanagedcontrolplane_controller.go b/controllers/azureasomanagedcontrolplane_controller.go index 362ffdc989e..3d3fbcfcb75 100644 --- a/controllers/azureasomanagedcontrolplane_controller.go +++ b/controllers/azureasomanagedcontrolplane_controller.go @@ -20,12 +20,16 @@ import ( "context" "errors" "fmt" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" "github.com/Azure/azure-service-operator/v2/pkg/genruntime" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/external" "sigs.k8s.io/cluster-api/util" @@ -52,6 +56,7 @@ var errInvalidClusterKind = errors.New("AzureASOManagedControlPlane cannot be us type AzureASOManagedControlPlaneReconciler struct { client.Client WatchFilterValue string + CredentialCache ASOCredentialCache newResourceReconciler func(*infrav1alpha.AzureASOManagedControlPlane, []*unstructured.Unstructured) resourceReconciler } @@ -241,21 +246,26 @@ func (r *AzureASOManagedControlPlaneReconciler) reconcileNormal(ctx context.Cont asoManagedControlPlane.Status.Version = "v" + *managedCluster.Status.CurrentKubernetesVersion } - err = r.reconcileKubeconfig(ctx, asoManagedControlPlane, cluster, managedCluster) + tokenExpiresIn, err := r.reconcileKubeconfig(ctx, asoManagedControlPlane, cluster, managedCluster) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err) } + if tokenExpiresIn != nil && *tokenExpiresIn <= 0 { // the token has already expired + return ctrl.Result{Requeue: true}, nil + } + // ensure we refresh the token when it expires + result := ctrl.Result{RequeueAfter: ptr.Deref(tokenExpiresIn, 0)} asoManagedControlPlane.Status.Ready = !asoManagedControlPlane.Status.ControlPlaneEndpoint.IsZero() // The AKS API doesn't allow us to distinguish between CAPI's definitions of "initialized" and "ready" so // we treat them equivalently. asoManagedControlPlane.Status.Initialized = asoManagedControlPlane.Status.Ready - return ctrl.Result{}, nil + return result, nil } -func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster, managedCluster *asocontainerservicev1.ManagedCluster) error { - ctx, _, done := tele.StartSpanWithLogger(ctx, +func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster, managedCluster *asocontainerservicev1.ManagedCluster) (*time.Duration, error) { + ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureASOManagedControlPlaneReconciler.reconcileKubeconfig", ) defer done() @@ -269,12 +279,50 @@ func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context. } } if secretRef == nil { - return reconcile.TerminalError(fmt.Errorf("ManagedCluster must define at least one of spec.operatorSpec.secrets.{userCredentials,adminCredentials}")) + return nil, reconcile.TerminalError(fmt.Errorf("ManagedCluster must define at least one of spec.operatorSpec.secrets.{userCredentials,adminCredentials}")) } asoKubeconfig := &corev1.Secret{} err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretRef.Name}, asoKubeconfig) if err != nil { - return fmt.Errorf("failed to fetch secret created by ASO: %w", err) + return nil, fmt.Errorf("failed to fetch secret created by ASO: %w", err) + } + + kubeconfigData := asoKubeconfig.Data[secretRef.Key] + var tokenExpiresIn *time.Duration + + if managedCluster.Status.AadProfile != nil && + ptr.Deref(managedCluster.Status.AadProfile.Managed, false) && + ptr.Deref(managedCluster.Status.DisableLocalAccounts, false) { + if secretRef.Name == secret.Name(cluster.Name, secret.Kubeconfig) { + return nil, fmt.Errorf("ASO-generated kubeconfig Secret name cannot be %q when local accounts are disabled on the ManagedCluster, CAPZ must be able to create and manage its own Secret with that name in order to augment the kubeconfig without conflicting with ASO", secretRef.Name) + } + + // Admin credentials cannot be retrieved when local accounts are disabled. Fetch a Bearer token like + // `kubelogin` would and set it in the kubeconfig to remove the need for that binary in CAPI controllers. + cred, err := r.CredentialCache.authTokenForASOResource(ctx, managedCluster) + if err != nil { + return nil, err + } + // magic string for AKS's managed Entra server ID: https://learn.microsoft.com/azure/aks/kubelogin-authentication#how-to-use-kubelogin-with-aks + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{"6dae42f8-4368-4678-94ff-3960e28e3630/.default"}}) + if err != nil { + return nil, err + } + tokenExpiresIn = ptr.To(time.Until(token.ExpiresOn)) + log.V(4).Info("retrieved access token", "expiresOn", token.ExpiresOn, "expiresIn", tokenExpiresIn) + + kubeconfig, err := clientcmd.Load(kubeconfigData) + if err != nil { + return nil, err + } + for _, a := range kubeconfig.AuthInfos { + a.Exec = nil + a.Token = token.Token + } + kubeconfigData, err = clientcmd.Write(*kubeconfig) + if err != nil { + return nil, err + } } expectedSecret := &corev1.Secret{ @@ -291,11 +339,15 @@ func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context. Labels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name}, }, Data: map[string][]byte{ - secret.KubeconfigDataName: asoKubeconfig.Data[secretRef.Key], + secret.KubeconfigDataName: kubeconfigData, }, } - return r.Patch(ctx, expectedSecret, client.Apply, client.FieldOwner("capz-manager"), client.ForceOwnership) + err = r.Patch(ctx, expectedSecret, client.Apply, client.FieldOwner("capz-manager"), client.ForceOwnership) + if err != nil { + return nil, err + } + return tokenExpiresIn, nil } func (r *AzureASOManagedControlPlaneReconciler) reconcilePaused(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane) (ctrl.Result, error) { diff --git a/controllers/azureasomanagedcontrolplane_controller_test.go b/controllers/azureasomanagedcontrolplane_controller_test.go index 1981dee1103..501227c51b4 100644 --- a/controllers/azureasomanagedcontrolplane_controller_test.go +++ b/controllers/azureasomanagedcontrolplane_controller_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" "github.com/Azure/azure-service-operator/v2/pkg/genruntime" . "github.com/onsi/gomega" @@ -31,6 +33,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" @@ -330,6 +334,277 @@ func TestAzureASOManagedControlPlaneReconcile(t *testing.T) { g.Expect(asoManagedControlPlane.Status.Ready).To(BeTrue()) }) + t.Run("successfully reconciles a kubeconfig with a token", func(t *testing.T) { + g := NewGomegaWithT(t) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: "ns", + }, + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &corev1.ObjectReference{ + APIVersion: infrav1alpha.GroupVersion.Identifier(), + Kind: infrav1alpha.AzureASOManagedClusterKind, + }, + }, + } + kubeconfig := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name(cluster.Name, secret.Kubeconfig) + "-user", + Namespace: cluster.Namespace, + }, + Data: map[string][]byte{ + "some other key": func() []byte { + kubeconfig := &clientcmdapi.Config{ + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + "some-user": { + Exec: &clientcmdapi.ExecConfig{}, + }, + }, + } + kubeconfigData, err := clientcmd.Write(*kubeconfig) + g.Expect(err).NotTo(HaveOccurred()) + return kubeconfigData + }(), + }, + } + managedCluster := &asocontainerservicev1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mc", + Namespace: cluster.Namespace, + }, + Spec: asocontainerservicev1.ManagedCluster_Spec{ + OperatorSpec: &asocontainerservicev1.ManagedClusterOperatorSpec{ + Secrets: &asocontainerservicev1.ManagedClusterOperatorSecrets{ + UserCredentials: &genruntime.SecretDestination{ + Name: secret.Name(cluster.Name, secret.Kubeconfig) + "-user", + Key: "some other key", + }, + }, + }, + }, + Status: asocontainerservicev1.ManagedCluster_STATUS{ + Fqdn: ptr.To("endpoint"), + AadProfile: &asocontainerservicev1.ManagedClusterAADProfile_STATUS{ + Managed: ptr.To(true), + }, + DisableLocalAccounts: ptr.To(true), + }, + } + asoManagedControlPlane := &infrav1alpha.AzureASOManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amcp", + Namespace: cluster.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.Identifier(), + Kind: "Cluster", + Name: cluster.Name, + }, + }, + Finalizers: []string{ + infrav1alpha.AzureASOManagedControlPlaneFinalizer, + }, + Annotations: map[string]string{ + clusterctlv1.BlockMoveAnnotation: "true", + }, + }, + Spec: infrav1alpha.AzureASOManagedControlPlaneSpec{ + AzureASOManagedControlPlaneTemplateResourceSpec: infrav1alpha.AzureASOManagedControlPlaneTemplateResourceSpec{ + Resources: []runtime.RawExtension{ + { + Raw: mcJSON(g, &asocontainerservicev1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedCluster.Name, + }, + }), + }, + }, + }, + }, + Status: infrav1alpha.AzureASOManagedControlPlaneStatus{ + Ready: false, + }, + } + c := fakeClientBuilder(). + WithObjects(cluster, asoManagedControlPlane, managedCluster, kubeconfig). + Build() + kubeConfigPatched := false + r := &AzureASOManagedControlPlaneReconciler{ + Client: &FakeClient{ + Client: c, + patchFunc: func(_ context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) error { + kubeconfigSecret := obj.(*corev1.Secret) + g.Expect(kubeconfigSecret.Data[secret.KubeconfigDataName]).NotTo(BeEmpty()) + kubeConfigPatched = true + + kubeconfig, err := clientcmd.Load(kubeconfigSecret.Data[secret.KubeconfigDataName]) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kubeconfig.AuthInfos).To(HaveEach(Satisfy(func(user *clientcmdapi.AuthInfo) bool { + return user.Exec == nil && + user.Token == "token" + }))) + + return nil + }, + }, + newResourceReconciler: func(_ *infrav1alpha.AzureASOManagedControlPlane, _ []*unstructured.Unstructured) resourceReconciler { + return &fakeResourceReconciler{ + reconcileFunc: func(ctx context.Context, o client.Object) error { + return nil + }, + } + }, + CredentialCache: fakeASOCredentialCache(func(_ context.Context, _ client.Object) (azcore.TokenCredential, error) { + return fakeTokenCredential{token: "token", expiresOn: time.Now().Add(1 * time.Hour)}, nil + }), + } + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(asoManagedControlPlane)}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Requeue).To(BeFalse()) + g.Expect(result.RequeueAfter).NotTo(BeZero()) + + g.Expect(c.Get(ctx, client.ObjectKeyFromObject(asoManagedControlPlane), asoManagedControlPlane)).To(Succeed()) + g.Expect(kubeConfigPatched).To(BeTrue()) + g.Expect(asoManagedControlPlane.Status.Ready).To(BeTrue()) + }) + + t.Run("successfully reconciles a kubeconfig with a token that has expired", func(t *testing.T) { + g := NewGomegaWithT(t) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: "ns", + }, + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &corev1.ObjectReference{ + APIVersion: infrav1alpha.GroupVersion.Identifier(), + Kind: infrav1alpha.AzureASOManagedClusterKind, + }, + }, + } + kubeconfig := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name(cluster.Name, secret.Kubeconfig) + "-user", + Namespace: cluster.Namespace, + }, + Data: map[string][]byte{ + "some other key": func() []byte { + kubeconfig := &clientcmdapi.Config{ + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + "some-user": { + Exec: &clientcmdapi.ExecConfig{}, + }, + }, + } + kubeconfigData, err := clientcmd.Write(*kubeconfig) + g.Expect(err).NotTo(HaveOccurred()) + return kubeconfigData + }(), + }, + } + managedCluster := &asocontainerservicev1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mc", + Namespace: cluster.Namespace, + }, + Spec: asocontainerservicev1.ManagedCluster_Spec{ + OperatorSpec: &asocontainerservicev1.ManagedClusterOperatorSpec{ + Secrets: &asocontainerservicev1.ManagedClusterOperatorSecrets{ + UserCredentials: &genruntime.SecretDestination{ + Name: secret.Name(cluster.Name, secret.Kubeconfig) + "-user", + Key: "some other key", + }, + }, + }, + }, + Status: asocontainerservicev1.ManagedCluster_STATUS{ + Fqdn: ptr.To("endpoint"), + AadProfile: &asocontainerservicev1.ManagedClusterAADProfile_STATUS{ + Managed: ptr.To(true), + }, + DisableLocalAccounts: ptr.To(true), + }, + } + asoManagedControlPlane := &infrav1alpha.AzureASOManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amcp", + Namespace: cluster.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.Identifier(), + Kind: "Cluster", + Name: cluster.Name, + }, + }, + Finalizers: []string{ + infrav1alpha.AzureASOManagedControlPlaneFinalizer, + }, + Annotations: map[string]string{ + clusterctlv1.BlockMoveAnnotation: "true", + }, + }, + Spec: infrav1alpha.AzureASOManagedControlPlaneSpec{ + AzureASOManagedControlPlaneTemplateResourceSpec: infrav1alpha.AzureASOManagedControlPlaneTemplateResourceSpec{ + Resources: []runtime.RawExtension{ + { + Raw: mcJSON(g, &asocontainerservicev1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedCluster.Name, + }, + }), + }, + }, + }, + }, + Status: infrav1alpha.AzureASOManagedControlPlaneStatus{ + Ready: true, + }, + } + c := fakeClientBuilder(). + WithObjects(cluster, asoManagedControlPlane, managedCluster, kubeconfig). + Build() + kubeConfigPatched := false + r := &AzureASOManagedControlPlaneReconciler{ + Client: &FakeClient{ + Client: c, + patchFunc: func(_ context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) error { + kubeconfigSecret := obj.(*corev1.Secret) + g.Expect(kubeconfigSecret.Data[secret.KubeconfigDataName]).NotTo(BeEmpty()) + kubeConfigPatched = true + + kubeconfig, err := clientcmd.Load(kubeconfigSecret.Data[secret.KubeconfigDataName]) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kubeconfig.AuthInfos).To(HaveEach(Satisfy(func(user *clientcmdapi.AuthInfo) bool { + return user.Exec == nil && + user.Token == "token" + }))) + + return nil + }, + }, + newResourceReconciler: func(_ *infrav1alpha.AzureASOManagedControlPlane, _ []*unstructured.Unstructured) resourceReconciler { + return &fakeResourceReconciler{ + reconcileFunc: func(ctx context.Context, o client.Object) error { + return nil + }, + } + }, + CredentialCache: fakeASOCredentialCache(func(_ context.Context, _ client.Object) (azcore.TokenCredential, error) { + return fakeTokenCredential{token: "token", expiresOn: time.Now()}, nil + }), + } + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(asoManagedControlPlane)}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(Equal(ctrl.Result{Requeue: true})) + + g.Expect(c.Get(ctx, client.ObjectKeyFromObject(asoManagedControlPlane), asoManagedControlPlane)).To(Succeed()) + g.Expect(kubeConfigPatched).To(BeTrue()) + g.Expect(asoManagedControlPlane.Status.Ready).To(BeFalse()) + }) + t.Run("successfully reconciles pause", func(t *testing.T) { g := NewGomegaWithT(t) @@ -478,3 +753,18 @@ func mcJSON(g Gomega, mc *asocontainerservicev1.ManagedCluster) []byte { g.Expect(err).NotTo(HaveOccurred()) return j } + +type fakeASOCredentialCache func(context.Context, client.Object) (azcore.TokenCredential, error) + +func (t fakeASOCredentialCache) authTokenForASOResource(ctx context.Context, obj client.Object) (azcore.TokenCredential, error) { + return t(ctx, obj) +} + +type fakeTokenCredential struct { + token string + expiresOn time.Time +} + +func (t fakeTokenCredential) GetToken(_ context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: t.token, ExpiresOn: t.expiresOn}, nil +} diff --git a/docs/book/src/managed/asomanagedcluster.md b/docs/book/src/managed/asomanagedcluster.md index 398d5dd6639..eb7f36c3a00 100644 --- a/docs/book/src/managed/asomanagedcluster.md +++ b/docs/book/src/managed/asomanagedcluster.md @@ -52,3 +52,40 @@ Kubernetes API, thereby making CAPZ the thinnest possible translation layer betw This experiment will help inform CAPZ whether this pattern may be a candidate for a potential v2 API. This functionality is enabled by default and can be disabled with the `ASOAPI` feature flag (set by the `EXP_ASO_API` environment variable). Please try it out and offer any feedback! + +### Disable Local Accounts + +When [local accounts are disabled](https://learn.microsoft.com/en-us/azure/aks/manage-local-accounts-managed-azure-ad#disable-local-accounts), +like for [AKS Automatic](https://learn.microsoft.com/en-us/azure/aks/intro-aks-automatic) clusters, the +kubeconfig generated by AKS assumes clients have access to the [`kubelogin`](https://azure.github.io/kubelogin/) +utility locally to authenticate with Entra. This is not the case for clients like the Cluster API controllers +which need to access Nodes in the workload cluster. To allow those controllers access, CAPZ will augment the +kubeconfig from AKS to remove the `exec` plugin and add a `token` which is an Entra ID access token that +clients can handle natively by passing as an `Authorization: Bearer ...` token. CAPZ authenticates with Entra +using the same ASO credentials used to create the ManagedCluster resource, which might be any of the options +described in [ASO's documentation](https://azure.github.io/azure-service-operator/guide/authentication/credential-format/) +and must be assigned the [Azure Kubernetes Service RBAC Cluster Admin](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/containers#azure-kubernetes-service-rbac-cluster-admin) Role. + +When defining the embedded ManagedCluster in an AzureASOManagedControlPlane, ASO will fail to retrieve +`adminCredentials` when local accounts are disabled, so `userCredentials` must be specified instead. In order +to leave room for CAPZ to manage the canonical `${CLUSTER_NAME}-kubeconfig` secret well-known to Cluster API, +another name must be specified for this Secret to avoid CAPZ and ASO overwriting each other: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedControlPlane +metadata: + name: ${CLUSTER_NAME} +spec: + resources: + - apiVersion: containerservice.azure.com/v1api20231001 + kind: ManagedCluster + metadata: + name: ${CLUSTER_NAME} + spec: + operatorSpec: + secrets: + userCredentials: + name: ${CLUSTER_NAME}-user-kubeconfig # NOT ${CLUSTER_NAME}-kubeconfig + key: value +``` diff --git a/main.go b/main.go index 2c1b98e0bcd..2a37ade5a70 100644 --- a/main.go +++ b/main.go @@ -57,6 +57,7 @@ import ( infrav1alpha "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/controllers" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" infrav1controllersexp "sigs.k8s.io/cluster-api-provider-azure/exp/controllers" @@ -369,6 +370,8 @@ func main() { } func registerControllers(ctx context.Context, mgr manager.Manager) { + credCache := azure.NewCredentialCache() + machineCache, err := coalescing.NewRequestCache(debouncingTimer) if err != nil { setupLog.Error(err, "failed to build machineCache ReconcileCache") @@ -528,6 +531,7 @@ func registerControllers(ctx context.Context, mgr manager.Manager) { if err := (&controllers.AzureASOManagedControlPlaneReconciler{ Client: mgr.GetClient(), WatchFilterValue: watchFilterValue, + CredentialCache: controllers.NewASOCredentialCache(credCache, mgr.GetClient()), }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: azureClusterConcurrency}); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AzureASOManagedControlPlane") os.Exit(1)