From e0a57f40b085cc1a02e5989797a14d02a59ca212 Mon Sep 17 00:00:00 2001 From: Milan Zazrivec Date: Fri, 29 Nov 2024 21:06:17 +0100 Subject: [PATCH] ROSA: Support for OCM service account credentials --- .../src/topics/rosa/creating-a-cluster.md | 77 +++++++++-- pkg/rosa/client.go | 111 +++++++++++---- pkg/rosa/client_test.go | 126 ++++++++++++++++++ 3 files changed, 274 insertions(+), 40 deletions(-) create mode 100644 pkg/rosa/client_test.go diff --git a/docs/book/src/topics/rosa/creating-a-cluster.md b/docs/book/src/topics/rosa/creating-a-cluster.md index b28c19cbad..dd7dacdfc8 100644 --- a/docs/book/src/topics/rosa/creating-a-cluster.md +++ b/docs/book/src/topics/rosa/creating-a-cluster.md @@ -1,30 +1,83 @@ # Creating a ROSA cluster ## Permissions -CAPA controller requires an API token in order to be able to provision ROSA clusters: +### Authentication using service account credentials +CAPA controller requires service account credentials to be able to provision ROSA clusters: +1. Visit [https://console.redhat.com/iam/service-accounts](https://console.redhat.com/iam/service-accounts) and create a new service account. -1. Visit [https://console.redhat.com/openshift/token](https://console.redhat.com/openshift/token) to retrieve your API authentication token - -1. Create a credentials secret within the target namespace with the token to be referenced later by `ROSAControlePlane` +1. Create a new kubernetes secret with the service account credentials to be referenced later by `ROSAControlPlane` ```shell kubectl create secret generic rosa-creds-secret \ - --from-literal=ocmToken='eyJhbGciOiJIUzI1NiIsI....' \ - --from-literal=ocmApiUrl='https://api.openshift.com' + --from-literal=ocmClientID='....' \ + --from-literal=ocmClientSecret='eyJhbGciOiJIUzI1NiIsI....' \ + --from-literal=ocmApiUrl='https://api.openshift.com' ``` - Alternatively, you can edit CAPA controller deployment to provide the credentials: + Note: to consume the secret without the need to reference it from your `ROSAControlPlane`, name your secret as `rosa-creds-secret` and create it in the CAPA manager namespace (usually `capa-system`) ```shell - kubectl edit deployment -n capa-system capa-controller-manager + kubectl -n capa-system create secret generic rosa-creds-secret \ + --from-literal=ocmClientID='....' \ + --from-literal=ocmClientSecret='eyJhbGciOiJIUzI1NiIsI....' \ + --from-literal=ocmApiUrl='https://api.openshift.com' ``` - and add the following environment variables to the manager container: - ```yaml - env: + +### Authentication using SSO offline token (DEPRECATED) +The SSO offline token is being deprecated and it is recommended to use service account credentials instead, as described above. + +1. Visit https://console.redhat.com/openshift/token to retrieve your SSO offline authentication token + +1. Create a credentials secret within the target namespace with the token to be referenced later by `ROSAControlePlane` +```shell + kubectl create secret generic rosa-creds-secret \ + --from-literal=ocmToken='eyJhbGciOiJIUzI1NiIsI....' \ + --from-literal=ocmApiUrl='https://api.openshift.com' + ``` + +Alternatively, you can edit CAPA controller deployment to provide the credentials +```shell + kubectl edit deployment -n capa-system capa-controller-manager +``` +and add the following environment variables to the manager container +```yaml + env: - name: OCM_TOKEN value: "" - name: OCM_API_URL value: "https://api.openshift.com" # or https://api.stage.openshift.com - ``` +``` + +### Migration from offline token to service account authentication + +1. Visit [https://console.redhat.com/iam/service-accounts](https://console.redhat.com/iam/service-accounts) and create a new service account. + +1. If you previously used kubernetes secret to specify the OCM credentials secret, edit the secret: +```shell + kubectl edit secret rosa-creds-secret +``` +where you will remove the `ocmToken` credentials and add base64 encoded `ocmClientID` and `ocmClientSecret` credentials like so: +```yaml +apiVersion: v1 + data: + ocmApiUrl: aHR0cHM6Ly9hcGkub3BlbnNoaWZ0LmNvbQ== + ocmClientID: Y2xpZW50X2lk... + ocmClientSecret: Y2xpZW50X3NlY3JldA==... + kind: Secret + type: Opaque +``` + +1. If you previously used capa manager deployment to specify the OCM offline token as environment variable, edit the manager deployment: +```shell + kubectl -n capa-system edit deployment capa-controller-manager +``` +and remove the `OCM_TOKEN` and `OCM_API_URL` variables, followed by `kubectl -n capa-system rollout restart deploy capa-controller-manager`. Then create the new default +secret in the `capa-system` namespace with: +```shell + kubectl -n capa-system create secret generic rosa-creds-secret \ + --from-literal=ocmClientID='....' \ + --from-literal=ocmClientSecret='eyJhbGciOiJIUzI1NiIsI....' \ + --from-literal=ocmApiUrl='https://api.openshift.com' +``` ## Prerequisites diff --git a/pkg/rosa/client.go b/pkg/rosa/client.go index 36c9ae333b..4aeebe170c 100644 --- a/pkg/rosa/client.go +++ b/pkg/rosa/client.go @@ -10,45 +10,67 @@ import ( ocmcfg "github.com/openshift/rosa/pkg/config" "github.com/openshift/rosa/pkg/ocm" "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/system" ) const ( - ocmTokenKey = "ocmToken" - ocmAPIURLKey = "ocmApiUrl" + ocmTokenKey = "ocmToken" + ocmAPIURLKey = "ocmApiUrl" + ocmClientIDKey = "ocmClientID" + ocmClientSecretKey = "ocmClientSecret" ) // NewOCMClient creates a new OCM client. func NewOCMClient(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (*ocm.Client, error) { - token, url, err := ocmCredentials(ctx, rosaScope) + token, url, clientID, clientSecret, err := ocmCredentials(ctx, rosaScope) if err != nil { return nil, err } - return ocm.NewClient().Logger(logrus.New()).Config(&ocmcfg.Config{ - AccessToken: token, - URL: url, - }).Build() + + ocmConfig := ocmcfg.Config{ + URL: url, + } + + if clientID != "" && clientSecret != "" { + ocmConfig.ClientID = clientID + ocmConfig.ClientSecret = clientSecret + } else if token != "" { + ocmConfig.AccessToken = token + } + + return ocm.NewClient().Logger(logrus.New()).Config(&ocmConfig).Build() } func newOCMRawConnection(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (*sdk.Connection, error) { - logger, err := sdk.NewGoLoggerBuilder(). + ocmSdkLogger, err := sdk.NewGoLoggerBuilder(). Debug(false). Build() if err != nil { return nil, fmt.Errorf("failed to build logger: %w", err) } - token, url, err := ocmCredentials(ctx, rosaScope) + + token, url, clientID, clientSecret, err := ocmCredentials(ctx, rosaScope) if err != nil { return nil, err } - connection, err := sdk.NewConnectionBuilder(). - Logger(logger). - Tokens(token). - URL(url). - Build() + connBuilder := sdk.NewConnectionBuilder(). + Logger(ocmSdkLogger). + URL(url) + + if clientID != "" && clientSecret != "" { + connBuilder.Client(clientID, clientSecret) + } else if token != "" { + connBuilder.Tokens(token) + } + + connection, err := connBuilder.Build() if err != nil { return nil, fmt.Errorf("failed to create ocm connection: %w", err) } @@ -56,28 +78,61 @@ func newOCMRawConnection(ctx context.Context, rosaScope *scope.ROSAControlPlaneS return connection, nil } -func ocmCredentials(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (string, string, error) { - var token string - var ocmAPIUrl string +func ocmCredentials(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (string, string, string, string, error) { + var token string // Offline SSO token + var ocmClientID string // Service account client id + var ocmClientSecret string // Service account client secret + var ocmAPIUrl string // https://api.openshift.com by default + var secret *corev1.Secret - secret := rosaScope.CredentialsSecret() + secret = rosaScope.CredentialsSecret() // We'll retrieve the OCM credentials ref from the ROSA control plane if secret != nil { if err := rosaScope.Client.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { - return "", "", fmt.Errorf("failed to get credentials secret: %w", err) + return "", "", "", "", fmt.Errorf("failed to get credentials secret: %w", err) } + } else { // If the reference to OCM secret wasn't specified in the ROSA control plane, we'll try to use a predefined secret name from the capa namespace + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rosa-creds-secret", + Namespace: system.GetManagerNamespace(), + }, + } + + err := rosaScope.Client.Get(ctx, client.ObjectKeyFromObject(secret), secret) + // We'll ignore non-existent secret so that we can try the ENV variable fallback below + // TODO: once the ENV variable fallback is gone, we can no longer ignore non-existent secret here + if err != nil && !apierrors.IsNotFound(err) { + return "", "", "", "", fmt.Errorf("failed to get credentials secret: %w", err) + } + } + + token = string(secret.Data[ocmTokenKey]) + ocmAPIUrl = string(secret.Data[ocmAPIURLKey]) + ocmClientID = string(secret.Data[ocmClientIDKey]) + ocmClientSecret = string(secret.Data[ocmClientSecretKey]) - token = string(secret.Data[ocmTokenKey]) - ocmAPIUrl = string(secret.Data[ocmAPIURLKey]) - } else { - // fallback to env variables if secrert is not set + // Deprecation warning in case SSO offline token was used + if token != "" { + rosaScope.Info("Using SSO offline token is deprecated, use service account credentials instead") + } + + if token == "" && (ocmClientID == "" || ocmClientSecret == "") { + // TODO: the ENV variables are to be removed with the next code release + // Last fall-back is to use OCM_TOKEN & OCM_API_URL environment variables (soon to be deprecated) token = os.Getenv("OCM_TOKEN") - if ocmAPIUrl = os.Getenv("OCM_API_URL"); ocmAPIUrl == "" { - ocmAPIUrl = "https://api.openshift.com" + ocmAPIUrl = os.Getenv("OCM_API_URL") + + if token != "" { + rosaScope.Info("Defining OCM credentials in environment variable is deprecated, use secret with service account credentials instead") + } else { + return "", "", "", "", + fmt.Errorf("OCM credentials have not been provided. Make sure to set the secret with service account credentials") } } - if token == "" { - return "", "", fmt.Errorf("token is not provided, be sure to set OCM_TOKEN env variable or reference a credentials secret with key %s", ocmTokenKey) + if ocmAPIUrl == "" { + ocmAPIUrl = "https://api.openshift.com" // Defaults to production URL } - return token, ocmAPIUrl, nil + + return token, ocmAPIUrl, ocmClientID, ocmClientSecret, nil } diff --git a/pkg/rosa/client_test.go b/pkg/rosa/client_test.go new file mode 100644 index 0000000000..023b9e220b --- /dev/null +++ b/pkg/rosa/client_test.go @@ -0,0 +1,126 @@ +package rosa + +import ( + "context" + "os" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + rosacontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/rosa/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/system" +) + +func createROSAControlPlaneScope(cp *rosacontrolplanev1.ROSAControlPlane, secrets ...*corev1.Secret) *scope.ROSAControlPlaneScope { + // k8s mock (fake) client + fakeClientBuilder := fake.NewClientBuilder() + for _, sec := range secrets { + fakeClientBuilder.WithObjects(sec) + } + + fakeClient := fakeClientBuilder.Build() + + // ROSA Control Plane Scope + rcpScope := &scope.ROSAControlPlaneScope{ + Client: fakeClient, + ControlPlane: cp, + Logger: *logger.NewLogger(klog.Background()), + } + + return rcpScope +} + +func createSecret(name, namespace, token, url, clientID, clientSecret string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "ocmToken": []byte(token), + "ocmApiUrl": []byte(url), + "ocmClientID": []byte(clientID), + "ocmClientSecret": []byte(clientSecret), + }, + } +} + +func createCP(namespace string) *rosacontrolplanev1.ROSAControlPlane { + return &rosacontrolplanev1.ROSAControlPlane{ + Spec: rosacontrolplanev1.RosaControlPlaneSpec{ + CredentialsSecretRef: &corev1.LocalObjectReference{ + Name: "rosa-creds-secret", + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + }, + } +} + +func TestOcmCredentials(t *testing.T) { + g := NewWithT(t) + + wlSecret := createSecret("rosa-creds-secret", "default", "", "url", "client-id", "client-secret") + mgrSecret := createSecret("rosa-creds-secret", system.GetManagerNamespace(), "", "url", "global-client-id", "global-client-secret") + + cp := createCP("default") + + // Test that ocmCredentials() prefers workload secret to global and environment secrets + cp.Spec = rosacontrolplanev1.RosaControlPlaneSpec{ + CredentialsSecretRef: &corev1.LocalObjectReference{ + Name: "rosa-creds-secret", + }, + } + os.Setenv("OCM_API_URL", "env-url") + os.Setenv("OCM_TOKEN", "env-token") + rcpScope := createROSAControlPlaneScope(cp, wlSecret, mgrSecret) + token, url, clientID, clientSecret, err := ocmCredentials(context.Background(), rcpScope) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(token).To(Equal(string(wlSecret.Data["ocmToken"]))) + g.Expect(url).To(Equal(string(wlSecret.Data["ocmApiUrl"]))) + g.Expect(clientID).To(Equal(string(wlSecret.Data["ocmClientID"]))) + g.Expect(clientSecret).To(Equal(string(wlSecret.Data["ocmClientSecret"]))) + + // Test that ocmCredentials() prefers global manager secret to environment secret in case workload secret is not specified + cp.Spec = rosacontrolplanev1.RosaControlPlaneSpec{} + rcpScope = createROSAControlPlaneScope(cp, wlSecret, mgrSecret) + token, url, clientID, clientSecret, err = ocmCredentials(context.Background(), rcpScope) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(token).To(Equal(string(mgrSecret.Data["ocmToken"]))) + g.Expect(url).To(Equal(string(mgrSecret.Data["ocmApiUrl"]))) + g.Expect(clientID).To(Equal(string(mgrSecret.Data["ocmClientID"]))) + g.Expect(clientSecret).To(Equal(string(mgrSecret.Data["ocmClientSecret"]))) + + // Test that ocmCredentials() returns environment secret in case workload and manager secret are not specified + cp.Spec = rosacontrolplanev1.RosaControlPlaneSpec{} + rcpScope = createROSAControlPlaneScope(cp) + token, url, clientID, clientSecret, err = ocmCredentials(context.Background(), rcpScope) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(token).To(Equal(os.Getenv("OCM_TOKEN"))) + g.Expect(url).To(Equal(os.Getenv("OCM_API_URL"))) + g.Expect(clientID).To(Equal("")) + g.Expect(clientSecret).To(Equal("")) + + // Test that ocmCredentials() returns error in case none of the secrets has been provided + cp.Spec = rosacontrolplanev1.RosaControlPlaneSpec{} + rcpScope = createROSAControlPlaneScope(cp) + os.Unsetenv("OCM_API_URL") + os.Unsetenv("OCM_TOKEN") + token, url, clientID, clientSecret, err = ocmCredentials(context.Background(), rcpScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(token).To(Equal("")) + g.Expect(url).To(Equal("")) + g.Expect(clientID).To(Equal("")) + g.Expect(clientSecret).To(Equal("")) +}