From cbdbda580f4c9b4ba789c9912f70519888175457 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Tue, 13 Feb 2024 22:21:45 +0200 Subject: [PATCH] Add support for boot, preRun and postRun commands on cloud-init --- apis/v1beta1/microk8sconfig_types.go | 14 +++++ apis/v1beta1/zz_generated.deepcopy.go | 15 +++++ bootstrap-components.yaml | 41 ++++++++++++++ ...trap.cluster.x-k8s.io_microk8sconfigs.yaml | 20 +++++++ ...ster.x-k8s.io_microk8sconfigtemplates.yaml | 21 +++++++ controllers/cloudinit/cloudinit.go | 3 +- .../cloudinit/cloudinit_common_test.go | 56 +++++++++++++++++++ controllers/cloudinit/controlplane_init.go | 11 +++- controllers/cloudinit/controlplane_join.go | 11 +++- controllers/cloudinit/worker_join.go | 11 +++- controllers/microk8sconfig_controller.go | 9 +++ 11 files changed, 208 insertions(+), 4 deletions(-) diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go index 534b415..46709bc 100644 --- a/apis/v1beta1/microk8sconfig_types.go +++ b/apis/v1beta1/microk8sconfig_types.go @@ -104,6 +104,20 @@ type InitConfiguration struct { // ExtraKubeletArgs is a list of extra arguments to add to the kubelet. // +optional ExtraKubeletArgs []string `json:"extraKubeletArgs,omitempty"` + + // BootCommands is a list of commands to run during boot. + // These will be injected into the `bootcmd` section of cloud-init. + BootCommands []string `json:"bootCommands,omitempty"` + + // PreRunCommands is a list of commands to run before installing MicroK8s. + // These will be injected into the `runcmd` section of cloud-init. + // +optional + PreRunCommands []string `json:"preRunCommands,omitempty"` + + // PostRunCommands is a list of commands to run after installing MicroK8s. + // These will be injected into the `runcmd` section of cloud-init. + // +optional + PostRunCommands []string `json:"postRunCommands,omitempty"` } // CloudInitWriteFile is a file that will be injected by cloud-init diff --git a/apis/v1beta1/zz_generated.deepcopy.go b/apis/v1beta1/zz_generated.deepcopy.go index d5a6d1a..2574619 100644 --- a/apis/v1beta1/zz_generated.deepcopy.go +++ b/apis/v1beta1/zz_generated.deepcopy.go @@ -84,6 +84,21 @@ func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.BootCommands != nil { + in, out := &in.BootCommands, &out.BootCommands + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PreRunCommands != nil { + in, out := &in.PreRunCommands, &out.PreRunCommands + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PostRunCommands != nil { + in, out := &in.PostRunCommands, &out.PostRunCommands + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitConfiguration. diff --git a/bootstrap-components.yaml b/bootstrap-components.yaml index dbbe59a..5c8dc19 100644 --- a/bootstrap-components.yaml +++ b/bootstrap-components.yaml @@ -85,6 +85,12 @@ spec: schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + bootCommands: + description: BootCommands is a list of commands to run during + boot. These will be injected into the `bootcmd` section of cloud-init. + items: + type: string + type: array confinement: description: The confinement (strict or classic) configuration enum: @@ -145,6 +151,20 @@ spec: noProxy: description: The optional no proxy configuration type: string + postRunCommands: + description: PostRunCommands is a list of commands to run after + installing MicroK8s. These will be injected into the `runcmd` + section of cloud-init. + items: + type: string + type: array + preRunCommands: + description: PreRunCommands is a list of commands to run before + installing MicroK8s. These will be injected into the `runcmd` + section of cloud-init. + items: + type: string + type: array riskLevel: default: stable description: The risk-level (stable, candidate, beta, or edge) @@ -336,6 +356,13 @@ spec: convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + bootCommands: + description: BootCommands is a list of commands to run + during boot. These will be injected into the `bootcmd` + section of cloud-init. + items: + type: string + type: array confinement: description: The confinement (strict or classic) configuration enum: @@ -397,6 +424,20 @@ spec: noProxy: description: The optional no proxy configuration type: string + postRunCommands: + description: PostRunCommands is a list of commands to + run after installing MicroK8s. These will be injected + into the `runcmd` section of cloud-init. + items: + type: string + type: array + preRunCommands: + description: PreRunCommands is a list of commands to run + before installing MicroK8s. These will be injected into + the `runcmd` section of cloud-init. + items: + type: string + type: array riskLevel: default: stable description: The risk-level (stable, candidate, beta, diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml index 4347e18..9678ad2 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml @@ -75,6 +75,12 @@ spec: schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + bootCommands: + description: BootCommands is a list of commands to run during + boot. These will be injected into the `bootcmd` section of cloud-init. + items: + type: string + type: array confinement: description: The confinement (strict or classic) configuration enum: @@ -135,6 +141,20 @@ spec: noProxy: description: The optional no proxy configuration type: string + postRunCommands: + description: PostRunCommands is a list of commands to run after + installing MicroK8s. These will be injected into the `runcmd` + section of cloud-init. + items: + type: string + type: array + preRunCommands: + description: PreRunCommands is a list of commands to run before + installing MicroK8s. These will be injected into the `runcmd` + section of cloud-init. + items: + type: string + type: array riskLevel: default: stable description: The risk-level (stable, candidate, beta, or edge) diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml index 45220e9..fae34e3 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml @@ -82,6 +82,13 @@ spec: convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + bootCommands: + description: BootCommands is a list of commands to run + during boot. These will be injected into the `bootcmd` + section of cloud-init. + items: + type: string + type: array confinement: description: The confinement (strict or classic) configuration enum: @@ -143,6 +150,20 @@ spec: noProxy: description: The optional no proxy configuration type: string + postRunCommands: + description: PostRunCommands is a list of commands to + run after installing MicroK8s. These will be injected + into the `runcmd` section of cloud-init. + items: + type: string + type: array + preRunCommands: + description: PreRunCommands is a list of commands to run + before installing MicroK8s. These will be injected into + the `runcmd` section of cloud-init. + items: + type: string + type: array riskLevel: default: stable description: The risk-level (stable, candidate, beta, diff --git a/controllers/cloudinit/cloudinit.go b/controllers/cloudinit/cloudinit.go index 7069c50..681e252 100644 --- a/controllers/cloudinit/cloudinit.go +++ b/controllers/cloudinit/cloudinit.go @@ -69,6 +69,7 @@ func NewBaseCloudConfig() *CloudConfig { }) } return &CloudConfig{ - WriteFiles: writeFiles, + WriteFiles: writeFiles, + RunCommands: []string{"set -x"}, } } diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go index 3bf154d..d2cf406 100644 --- a/controllers/cloudinit/cloudinit_common_test.go +++ b/controllers/cloudinit/cloudinit_common_test.go @@ -326,4 +326,60 @@ func TestCloudConfigInput(t *testing.T) { }) } }) + + t.Run("CustomCommands", func(t *testing.T) { + for _, tc := range []struct { + name string + makeCloudConfig func() (*cloudinit.CloudConfig, error) + }{ + { + name: "ControlPlaneInit", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + BootCommands: []string{"cmd1"}, + PreRunCommands: []string{"cmd2"}, + PostRunCommands: []string{"cmd3"}, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + { + name: "ControlPlaneJoin", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + BootCommands: []string{"cmd1"}, + PreRunCommands: []string{"cmd2"}, + PostRunCommands: []string{"cmd3"}, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + { + name: "Worker", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{ + BootCommands: []string{"cmd1"}, + PreRunCommands: []string{"cmd2"}, + PostRunCommands: []string{"cmd3"}, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c, err := tc.makeCloudConfig() + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(c.BootCommands).To(ConsistOf(`cmd1`)) + g.Expect(c.RunCommands).To(ContainElement(`cmd2`)) + g.Expect(c.RunCommands[len(c.RunCommands)-1]).To(Equal(`cmd3`)) + }) + } + }) } diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index 7e459b4..817658a 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -69,6 +69,12 @@ type ControlPlaneInitInput struct { SnapstoreHTTPProxy string // SnapstoreHTTPSProxy is https_proxy configuration for snap store. SnapstoreHTTPSProxy string + // BootCommands is a list of commands to add to the "bootcmd" section of cloud-init. + BootCommands []string + // PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s. + PreRunCommands []string + // PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s. + PostRunCommands []string } func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { @@ -128,8 +134,10 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { Owner: "root:root", }) } + cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...) + + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...) cloudConfig.RunCommands = append(cloudConfig.RunCommands, - "set -x", fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy), fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId), scriptPath(disableHostServicesScript), @@ -146,6 +154,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { fmt.Sprintf("%s %s", scriptPath(microk8sEnableScript), strings.Join(addons, " ")), fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", input.TokenTTL, input.Token), ) + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...) return cloudConfig, nil } diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 85431d7..59fb3f4 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -65,6 +65,12 @@ type ControlPlaneJoinInput struct { SnapstoreHTTPProxy string // SnapstoreHTTPSProxy is https_proxy configuration for snap store. SnapstoreHTTPSProxy string + // BootCommands is a list of commands to add to the "bootcmd" section of cloud-init. + BootCommands []string + // PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s. + PreRunCommands []string + // PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s. + PostRunCommands []string } func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { @@ -108,8 +114,10 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token) joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token) + cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...) + + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...) cloudConfig.RunCommands = append(cloudConfig.RunCommands, - "set -x", fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy), fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId), scriptPath(disableHostServicesScript), @@ -126,6 +134,7 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { scriptPath(configureAPIServerScript), fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", input.TokenTTL, input.Token), ) + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...) return cloudConfig, nil } diff --git a/controllers/cloudinit/worker_join.go b/controllers/cloudinit/worker_join.go index e2b96bc..498c38e 100644 --- a/controllers/cloudinit/worker_join.go +++ b/controllers/cloudinit/worker_join.go @@ -58,6 +58,12 @@ type WorkerInput struct { SnapstoreHTTPProxy string // SnapstoreHTTPSProxy is https_proxy configuration for snap store. SnapstoreHTTPSProxy string + // BootCommands is a list of commands to add to the "bootcmd" section of cloud-init. + BootCommands []string + // PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s. + PreRunCommands []string + // PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s. + PostRunCommands []string } func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { @@ -97,8 +103,10 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token) joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token) + cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...) + + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...) cloudConfig.RunCommands = append(cloudConfig.RunCommands, - "set -x", fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy), fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId), scriptPath(disableHostServicesScript), @@ -110,6 +118,7 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { fmt.Sprintf("%s yes %q %q", scriptPath(microk8sJoinScript), joinStr, joinStrAlt), fmt.Sprintf("%s %s 6443 %s", scriptPath(configureTraefikScript), input.ControlPlaneEndpoint, stopApiServerProxyRefreshes), ) + cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...) return cloudConfig, nil } diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index 9e75d59..5697de6 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -310,6 +310,9 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, SnapstoreHTTPProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy, SnapstoreHTTPSProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPSProxy, + BootCommands: microk8sConfig.Spec.InitConfiguration.BootCommands, + PreRunCommands: microk8sConfig.Spec.InitConfiguration.PreRunCommands, + PostRunCommands: microk8sConfig.Spec.InitConfiguration.PostRunCommands, } if controlPlaneInput.TokenTTL == 0 { controlPlaneInput.TokenTTL = 315569260 @@ -410,6 +413,9 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, SnapstoreHTTPProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy, SnapstoreHTTPSProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPSProxy, + BootCommands: microk8sConfig.Spec.InitConfiguration.BootCommands, + PreRunCommands: microk8sConfig.Spec.InitConfiguration.PreRunCommands, + PostRunCommands: microk8sConfig.Spec.InitConfiguration.PostRunCommands, } if controlPlaneInput.TokenTTL == 0 { controlPlaneInput.TokenTTL = 315569260 @@ -509,6 +515,9 @@ func (r *MicroK8sConfigReconciler) handleJoiningWorkerNode(ctx context.Context, workerInput.ExtraKubeletArgs = c.ExtraKubeletArgs workerInput.ExtraWriteFiles = cloudinit.WriteFilesFromAPI(c.ExtraWriteFiles) + workerInput.BootCommands = c.BootCommands + workerInput.PreRunCommands = c.PreRunCommands + workerInput.PostRunCommands = c.PostRunCommands } bootstrapInitData, err := cloudinit.NewJoinWorker(workerInput) if err != nil {