From 1572fb15268434994b21acb2d70c2efde6e45e86 Mon Sep 17 00:00:00 2001 From: "Homayoon (Hue) Alimohammadi" Date: Wed, 9 Oct 2024 18:03:23 +0400 Subject: [PATCH 1/5] Add capi-auth-token file to control plane machines --- controllers/cloudinit/cloudinit.go | 5 + controllers/cloudinit/controlplane_init.go | 3 + .../cloudinit/controlplane_init_test.go | 8 ++ controllers/cloudinit/controlplane_join.go | 8 ++ .../cloudinit/controlplane_join_test.go | 11 +++ controllers/microk8sconfig_controller.go | 18 ++++ pkg/token/token.go | 99 +++++++++++++++++++ 7 files changed, 152 insertions(+) create mode 100644 pkg/token/token.go diff --git a/controllers/cloudinit/cloudinit.go b/controllers/cloudinit/cloudinit.go index 681e252..5319133 100644 --- a/controllers/cloudinit/cloudinit.go +++ b/controllers/cloudinit/cloudinit.go @@ -19,9 +19,14 @@ package cloudinit import ( "bytes" "fmt" + "path/filepath" "text/template" ) +var ( + CAPIAuthTokenPath = filepath.Join("/capi", "etc", "token") +) + // File is a file that cloud-init will create. type File struct { // Content of the file to create. diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index cfb5515..d9ff6cb 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -27,6 +27,8 @@ import ( // ControlPlaneInitInput defines the context needed to generate a controlplane instance to init a cluster. type ControlPlaneInitInput struct { + // AuthToken will be used for authenticating CAPI requests. + AuthToken string // CAKey is the PEM-encoded key of the cluster CA certificate. CAKey string // CACert is the PEM-encoded cert of the cluster CA certificate. @@ -131,6 +133,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { cloudConfig.WriteFiles, File{Content: input.CAKey, Path: filepath.Join("/var", "tmp", "ca.key"), Permissions: "0600", Owner: "root:root"}, File{Content: input.CACert, Path: filepath.Join("/var", "tmp", "ca.crt"), Permissions: "0600", Owner: "root:root"}, + File{Content: input.AuthToken, Path: CAPIAuthTokenPath, Permissions: "0600", Owner: "root:root"}, ) cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index 36e7849..9031db2 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -28,7 +28,9 @@ func TestControlPlaneInit(t *testing.T) { t.Run("Simple", func(t *testing.T) { g := NewWithT(t) + authToken := "capi-auth-token" cloudConfig, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + AuthToken: authToken, CAKey: `CA KEY DATA`, CACert: `CA CERT DATA`, ControlPlaneEndpoint: "k8s.my-domain.com", @@ -75,6 +77,12 @@ func TestControlPlaneInit(t *testing.T) { Permissions: "0600", Owner: "root:root", }, + cloudinit.File{ + Content: authToken, + Path: cloudinit.CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }, )) _, err = cloudinit.GenerateCloudConfig(cloudConfig) diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 56362ea..fd2545f 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -27,6 +27,8 @@ import ( // ControlPlaneJoinInput defines the context needed to generate a controlplane instance to join a cluster. type ControlPlaneJoinInput struct { + // AuthToken will be used for authenticating CAPI requests. + AuthToken string // ControlPlaneEndpoint is the control plane endpoint of the cluster. ControlPlaneEndpoint string // Token is the token that will be used for joining other nodes to the cluster. @@ -109,6 +111,12 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { } cloudConfig := NewBaseCloudConfig() + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ + Content: input.AuthToken, + Path: CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }) cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) if args := input.ExtraKubeletArgs; len(args) > 0 { cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go index aa31202..0d98a57 100644 --- a/controllers/cloudinit/controlplane_join_test.go +++ b/controllers/cloudinit/controlplane_join_test.go @@ -28,7 +28,9 @@ func TestControlPlaneJoin(t *testing.T) { t.Run("Simple", func(t *testing.T) { g := NewWithT(t) + authToken := "capi-auth-token" cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + AuthToken: authToken, ControlPlaneEndpoint: "k8s.my-domain.com", KubernetesVersion: "v1.25.2", ClusterAgentPort: "30000", @@ -60,6 +62,15 @@ func TestControlPlaneJoin(t *testing.T) { `microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, })) + g.Expect(cloudConfig.WriteFiles).To(ContainElements( + cloudinit.File{ + Content: authToken, + Path: cloudinit.CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }, + )) + _, err = cloudinit.GenerateCloudConfig(cloudConfig) g.Expect(err).ToNot(HaveOccurred()) }) diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index c4fb761..9acb506 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -59,6 +59,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/source" + + tokenpkg "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token" ) type InitLocker interface { @@ -213,6 +215,10 @@ func (r *MicroK8sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } + if err = tokenpkg.Reconcile(ctx, r.Client, util.ObjectKey(scope.Cluster)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile token: %w", err) + } + // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { log.Info("Cluster control plane is not initialized, waiting") @@ -296,7 +302,13 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte portOfDqlite = remappedDqlitePort } + authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster)) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err) + } + controlPlaneInput := &cloudinit.ControlPlaneInitInput{ + AuthToken: authToken, CACert: *cert, CAKey: *key, ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, @@ -403,7 +415,13 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster)) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err) + } + controlPlaneInput := &cloudinit.ControlPlaneJoinInput{ + AuthToken: authToken, ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, Token: token, TokenTTL: microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs, diff --git a/pkg/token/token.go b/pkg/token/token.go new file mode 100644 index 0000000..a536c21 --- /dev/null +++ b/pkg/token/token.go @@ -0,0 +1,99 @@ +package token + +import ( + "context" + cryptorand "crypto/rand" + "encoding/base64" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + AuthTokenNameSuffix = "capi-auth-token" +) + +// Reconcile ensures that a token secret exists for the given cluster. +func Reconcile(ctx context.Context, c client.Client, clusterKey client.ObjectKey) error { + if _, err := getSecret(ctx, c, clusterKey); err != nil { + if apierrors.IsNotFound(err) { + if _, err := generateAndStore(ctx, c, clusterKey); err != nil { + return fmt.Errorf("failed to generate and store token: %w", err) + } + return nil + } + } + + return nil +} + +// Lookup retrieves the token for the given cluster. +func Lookup(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (string, error) { + secret, err := getSecret(ctx, c, clusterKey) + if err != nil { + return "", fmt.Errorf("failed to get secret: %w", err) + } + + v, ok := secret.Data["token"] + if !ok { + return "", fmt.Errorf("token not found in secret") + } + + return string(v), nil +} + +// name returns the name of the token secret, computed by convention using the name of the cluster. +func authTokenName(clusterName string) string { + return fmt.Sprintf("%s-%s", clusterName, AuthTokenNameSuffix) +} + +// getSecret retrieves the token secret for the given cluster. +func getSecret(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: authTokenName(clusterKey.Name), + Namespace: clusterKey.Namespace, + } + if err := c.Get(ctx, key, s); err != nil { + return nil, fmt.Errorf("failed to get secret: %w", err) + } + + return s, nil +} + +// generateAndStore generates a new token and stores it in a secret. +func generateAndStore(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) { + token, err := randomB64(16) + if err != nil { + return nil, fmt.Errorf("failed to generate token: %v", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterKey.Namespace, + Name: authTokenName(clusterKey.Name), + }, + StringData: map[string]string{ + "token": token, + }, + } + + if err := c.Create(ctx, secret); err != nil { + return nil, fmt.Errorf("failed to create secret: %v", err) + } + + return secret, nil +} + +// randomB64 generates a random base64 string of n bytes. +func randomB64(n int) (string, error) { + b := make([]byte, n) + _, err := cryptorand.Read(b) + if err != nil { + return "", fmt.Errorf("failed to read random bytes: %v", err) + } + return base64.StdEncoding.EncodeToString(b), nil +} From c94ccf3d4f36fd339c03ec2f1a7d4bd5b6faad98 Mon Sep 17 00:00:00 2001 From: "Homayoon (Hue) Alimohammadi" Date: Wed, 9 Oct 2024 18:15:14 +0400 Subject: [PATCH 2/5] Fix lint --- pkg/token/token.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/token/token.go b/pkg/token/token.go index a536c21..2d6e225 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -68,7 +68,7 @@ func getSecret(ctx context.Context, c client.Client, clusterKey client.ObjectKey func generateAndStore(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) { token, err := randomB64(16) if err != nil { - return nil, fmt.Errorf("failed to generate token: %v", err) + return nil, fmt.Errorf("failed to generate token: %w", err) } secret := &corev1.Secret{ @@ -82,7 +82,7 @@ func generateAndStore(ctx context.Context, c client.Client, clusterKey client.Ob } if err := c.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret: %v", err) + return nil, fmt.Errorf("failed to create secret: %w", err) } return secret, nil @@ -93,7 +93,7 @@ func randomB64(n int) (string, error) { b := make([]byte, n) _, err := cryptorand.Read(b) if err != nil { - return "", fmt.Errorf("failed to read random bytes: %v", err) + return "", fmt.Errorf("failed to read random bytes: %w", err) } return base64.StdEncoding.EncodeToString(b), nil } From 57614f3762a0153a6971867881391ba3511aee6b Mon Sep 17 00:00:00 2001 From: "Homayoon (Hue) Alimohammadi" Date: Thu, 10 Oct 2024 16:33:38 +0400 Subject: [PATCH 3/5] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e8eb869..3c14177 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN go mod download COPY main.go main.go COPY apis/ apis/ COPY controllers/ controllers/ +COPY pkg/ pkg/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -a -ldflags '-s -w' -o manager main.go From dead100b23d80bf84eb35aaef74e6898d2d2a7e9 Mon Sep 17 00:00:00 2001 From: "Homayoon (Hue) Alimohammadi" Date: Fri, 11 Oct 2024 10:45:13 +0400 Subject: [PATCH 4/5] Address comments and issues --- controllers/cloudinit/controlplane_init.go | 2 +- controllers/cloudinit/controlplane_join.go | 2 +- pkg/token/token.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index d9ff6cb..9dde04b 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -27,7 +27,7 @@ import ( // ControlPlaneInitInput defines the context needed to generate a controlplane instance to init a cluster. type ControlPlaneInitInput struct { - // AuthToken will be used for authenticating CAPI requests. + // AuthToken will be used for authenticating CAPI-only requests to the cluster-agent. AuthToken string // CAKey is the PEM-encoded key of the cluster CA certificate. CAKey string diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index fd2545f..0ac4492 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -27,7 +27,7 @@ import ( // ControlPlaneJoinInput defines the context needed to generate a controlplane instance to join a cluster. type ControlPlaneJoinInput struct { - // AuthToken will be used for authenticating CAPI requests. + // AuthToken will be used for authenticating CAPI-only requests to the cluster-agent. AuthToken string // ControlPlaneEndpoint is the control plane endpoint of the cluster. ControlPlaneEndpoint string diff --git a/pkg/token/token.go b/pkg/token/token.go index 2d6e225..485f802 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -45,7 +45,7 @@ func Lookup(ctx context.Context, c client.Client, clusterKey client.ObjectKey) ( return string(v), nil } -// name returns the name of the token secret, computed by convention using the name of the cluster. +// authTokenName returns the name of the auth-token secret, computed by convention using the name of the cluster. func authTokenName(clusterName string) string { return fmt.Sprintf("%s-%s", clusterName, AuthTokenNameSuffix) } From 85ce23b9181f53778cb2dbf2cf5f2072d7978509 Mon Sep 17 00:00:00 2001 From: "Homayoon (Hue) Alimohammadi" Date: Fri, 11 Oct 2024 11:45:58 +0400 Subject: [PATCH 5/5] Add test for token pkg --- pkg/token/token.go | 6 ++- pkg/token/token_test.go | 86 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 pkg/token/token_test.go diff --git a/pkg/token/token.go b/pkg/token/token.go index 485f802..4feee74 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -76,9 +77,10 @@ func generateAndStore(ctx context.Context, c client.Client, clusterKey client.Ob Namespace: clusterKey.Namespace, Name: authTokenName(clusterKey.Name), }, - StringData: map[string]string{ - "token": token, + Data: map[string][]byte{ + "token": []byte(token), }, + Type: clusterv1.ClusterSecretType, } if err := c.Create(ctx, secret); err != nil { diff --git a/pkg/token/token_test.go b/pkg/token/token_test.go new file mode 100644 index 0000000..419b05f --- /dev/null +++ b/pkg/token/token_test.go @@ -0,0 +1,86 @@ +package token_test + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token" +) + +func TestReconcile(t *testing.T) { + t.Run("SecretAvailableSucceeds", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + }, + } + c := fake.NewClientBuilder().WithObjects(secret).Build() + + g := NewWithT(t) + + g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed()) + }) + + t.Run("SecretNotFoundGenerates", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + c := fake.NewClientBuilder().Build() + + g := NewWithT(t) + + g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed()) + + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + } + g.Expect(c.Get(context.Background(), key, s)).To(Succeed()) + g.Expect(s.ObjectMeta.Name).To(Equal(fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix))) + g.Expect(s.ObjectMeta.Namespace).To(Equal(namespace)) + g.Expect(string(s.Data["token"])).ToNot(BeEmpty()) + }) + + t.Run("LookupFailsIfNoSecret", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + c := fake.NewClientBuilder().Build() + + g := NewWithT(t) + + _, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace}) + g.Expect(err).To(HaveOccurred()) + }) + + t.Run("LookupSucceedsIfSecretExists", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + expToken := "test-token" + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + }, + Data: map[string][]byte{ + "token": []byte(expToken), + }, + } + c := fake.NewClientBuilder().WithObjects(secret).Build() + + g := NewWithT(t) + + token, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(token).To(Equal(expToken)) + }) +}