From 0e64a26d6b4d6ea7483f5daa713e677c42ff87f2 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 28 Nov 2022 17:31:05 +0200 Subject: [PATCH 1/3] Refactor confinement and add risk-level tests --- .../cloudinit/cloudinit_common_test.go | 45 ++++++++++------ .../cloudinit/controlplane_init_test.go | 53 ------------------- .../cloudinit/controlplane_join_test.go | 37 ------------- controllers/cloudinit/worker_join_test.go | 28 ---------- 4 files changed, 29 insertions(+), 134 deletions(-) diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go index a5a0e1e..dbeacbf 100644 --- a/controllers/cloudinit/cloudinit_common_test.go +++ b/controllers/cloudinit/cloudinit_common_test.go @@ -17,6 +17,7 @@ limitations under the License. package cloudinit_test import ( + "fmt" "strings" "testing" @@ -29,53 +30,65 @@ func TestCloudConfigInput(t *testing.T) { t.Run("ChannelAndRiskLevel", func(t *testing.T) { for _, tc := range []struct { name string - makeCloudConfig func() (*cloudinit.CloudConfig, error) + makeCloudConfig func(confinement, risk string) (*cloudinit.CloudConfig, error) }{ { name: "ControlPlaneInit", - makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) { return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ KubernetesVersion: "v1.25.0", + Confinement: confinement, Token: strings.Repeat("a", 32), TokenTTL: 100, - Confinement: "strict", - RiskLevel: "edge", + RiskLevel: risk, }) }, }, { name: "ControlPlaneJoin", - makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) { return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ KubernetesVersion: "v1.25.0", + Confinement: confinement, Token: strings.Repeat("a", 32), TokenTTL: 100, - Confinement: "strict", - RiskLevel: "edge", + RiskLevel: risk, }) }, }, { name: "Worker", - makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) { return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{ KubernetesVersion: "v1.25.0", + Confinement: confinement, Token: strings.Repeat("a", 32), - Confinement: "strict", - RiskLevel: "edge", + RiskLevel: risk, }) }, }, } { t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - c, err := tc.makeCloudConfig() - g.Expect(err).NotTo(HaveOccurred()) + for _, confinement := range []string{"strict", "classic"} { + t.Run(confinement, func(t *testing.T) { + for _, risk := range []string{"stable", "candidate", "beta", "edge"} { + t.Run(risk, func(t *testing.T) { + g := NewWithT(t) + c, err := tc.makeCloudConfig(confinement, risk) + g.Expect(err).NotTo(HaveOccurred()) - g.Expect(c.RunCommands).To(ContainElement(`/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict/edge"`)) + if confinement == "classic" { + g.Expect(c.RunCommands).To(ContainElement(fmt.Sprintf(`/capi-scripts/00-install-microk8s.sh "--channel 1.25/%s --classic"`, risk))) + } else { + g.Expect(c.RunCommands).To(ContainElement(fmt.Sprintf(`/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict/%s"`, risk))) + } - _, err = cloudinit.GenerateCloudConfig(c) - g.Expect(err).NotTo(HaveOccurred()) + _, err = cloudinit.GenerateCloudConfig(c) + g.Expect(err).NotTo(HaveOccurred()) + }) + } + }) + } }) } }) diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index 8897722..71fcacf 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -75,56 +75,3 @@ func TestControlPlaneInit(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) }) } - -func TestConfinementControlPlaneInit(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - g := NewWithT(t) - - cloudConfig, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ - CAKey: `CA KEY DATA`, - CACert: `CA CERT DATA`, - ControlPlaneEndpoint: "k8s.my-domain.com", - KubernetesVersion: "v1.25.2", - ClusterAgentPort: "30000", - DqlitePort: "2379", - IPinIP: true, - Token: strings.Repeat("a", 32), - TokenTTL: 10000, - Confinement: "strict", - }) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(cloudConfig.RunCommands).To(Equal([]string{ - `set -x`, - `/capi-scripts/00-disable-host-services.sh`, - `/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict"`, - `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, - `microk8s status --wait-ready`, - `microk8s refresh-certs /var/tmp`, - `/capi-scripts/10-configure-calico-ipip.sh true`, - `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, - `/capi-scripts/10-configure-dqlite-port.sh "2379"`, - `/capi-scripts/10-configure-apiserver.sh "DNS" "k8s.my-domain.com"`, - `/capi-scripts/20-microk8s-enable.sh "dns"`, - `microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, - })) - - g.Expect(cloudConfig.WriteFiles).To(ContainElements( - cloudinit.File{ - Content: "CA KEY DATA", - Path: "/var/tmp/ca.key", - Permissions: "0600", - Owner: "root:root", - }, - cloudinit.File{ - Content: "CA CERT DATA", - Path: "/var/tmp/ca.crt", - Permissions: "0600", - Owner: "root:root", - }, - )) - - _, err = cloudinit.GenerateCloudConfig(cloudConfig) - g.Expect(err).ToNot(HaveOccurred()) - }) -} diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go index 81b5597..89fb61a 100644 --- a/controllers/cloudinit/controlplane_join_test.go +++ b/controllers/cloudinit/controlplane_join_test.go @@ -59,40 +59,3 @@ func TestControlPlaneJoin(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) }) } - -func TestConfinementControlPlaneJoin(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - g := NewWithT(t) - - cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ - ControlPlaneEndpoint: "k8s.my-domain.com", - KubernetesVersion: "v1.25.2", - ClusterAgentPort: "30000", - DqlitePort: "2379", - IPinIP: true, - Token: strings.Repeat("a", 32), - TokenTTL: 10000, - JoinNodeIP: "10.0.3.39", - Confinement: "strict", - }) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(cloudConfig.RunCommands).To(Equal([]string{ - `set -x`, - `/capi-scripts/00-disable-host-services.sh`, - `/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict"`, - `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, - `microk8s status --wait-ready`, - `/capi-scripts/10-configure-calico-ipip.sh true`, - `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, - `/capi-scripts/10-configure-dqlite-port.sh "2379"`, - `microk8s status --wait-ready`, - `/capi-scripts/20-microk8s-join.sh "10.0.3.39:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, - `/capi-scripts/10-configure-apiserver.sh "DNS" "k8s.my-domain.com"`, - `microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, - })) - - _, err = cloudinit.GenerateCloudConfig(cloudConfig) - g.Expect(err).ToNot(HaveOccurred()) - }) -} diff --git a/controllers/cloudinit/worker_join_test.go b/controllers/cloudinit/worker_join_test.go index 8088b44..eff279e 100644 --- a/controllers/cloudinit/worker_join_test.go +++ b/controllers/cloudinit/worker_join_test.go @@ -50,31 +50,3 @@ func TestWorkerJoin(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) }) } - -func TestConfinementWorkerJoin(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - g := NewWithT(t) - - cloudConfig, err := cloudinit.NewJoinWorker(&cloudinit.WorkerInput{ - KubernetesVersion: "v1.25.3", - ClusterAgentPort: "30000", - Token: strings.Repeat("a", 32), - JoinNodeIP: "10.0.3.194", - Confinement: "strict", - }) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(cloudConfig.RunCommands).To(Equal([]string{ - `set -x`, - `/capi-scripts/00-disable-host-services.sh`, - `/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict"`, - `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, - `microk8s status --wait-ready`, - `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, - `/capi-scripts/20-microk8s-join.sh "10.0.3.194:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" --worker`, - })) - - _, err = cloudinit.GenerateCloudConfig(cloudConfig) - g.Expect(err).ToNot(HaveOccurred()) - }) -} From 45bae458873294e46b860f8573b2e01f163c56d5 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 28 Nov 2022 18:45:52 +0200 Subject: [PATCH 2/3] Add ExtraKubeletArgs and regenerate manifests --- apis/v1beta1/microk8sconfig_types.go | 4 + apis/v1beta1/zz_generated.deepcopy.go | 25 ++++++ bootstrap-components.yaml | 88 +++++++++++++++++++ ...trap.cluster.x-k8s.io_microk8sconfigs.yaml | 6 ++ ...ster.x-k8s.io_microk8sconfigtemplates.yaml | 6 ++ 5 files changed, 129 insertions(+) diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go index 0203268..752e088 100644 --- a/apis/v1beta1/microk8sconfig_types.go +++ b/apis/v1beta1/microk8sconfig_types.go @@ -84,6 +84,10 @@ type InitConfiguration struct { // ExtraWriteFiles is a list of extra files to inject with cloud-init. // +optional ExtraWriteFiles []CloudInitWriteFile `json:"extraWriteFiles,omitempty"` + + // ExtraKubeletArgs is a list of extra arguments to add to the kubelet. + // +optional + ExtraKubeletArgs []string `json:"extraKubeletArgs,omitempty"` } // CloudInitWriteFile is a file that will be injected by cloud-init diff --git a/apis/v1beta1/zz_generated.deepcopy.go b/apis/v1beta1/zz_generated.deepcopy.go index 1f38cba..d5a6d1a 100644 --- a/apis/v1beta1/zz_generated.deepcopy.go +++ b/apis/v1beta1/zz_generated.deepcopy.go @@ -26,6 +26,21 @@ import ( apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudInitWriteFile) DeepCopyInto(out *CloudInitWriteFile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudInitWriteFile. +func (in *CloudInitWriteFile) DeepCopy() *CloudInitWriteFile { + if in == nil { + return nil + } + out := new(CloudInitWriteFile) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { *out = *in @@ -59,6 +74,16 @@ func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ExtraWriteFiles != nil { + in, out := &in.ExtraWriteFiles, &out.ExtraWriteFiles + *out = make([]CloudInitWriteFile, len(*in)) + copy(*out, *in) + } + if in.ExtraKubeletArgs != nil { + in, out := &in.ExtraKubeletArgs, &out.ExtraKubeletArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitConfiguration. diff --git a/bootstrap-components.yaml b/bootstrap-components.yaml index b29b132..58f9ee2 100644 --- a/bootstrap-components.yaml +++ b/bootstrap-components.yaml @@ -57,9 +57,24 @@ spec: type: object initConfiguration: properties: + Confinement: + description: The confinement (strict or classic) configuration + enum: + - classic + - strict + type: string IPinIP: description: The optional IPinIP configuration type: boolean + RiskLevel: + default: stable + description: The risk-level (stable, candidate, beta, or edge) for the snaps + enum: + - stable + - candidate + - beta + - edge + type: string addons: description: List of addons to be enabled upon cluster creation items: @@ -68,6 +83,35 @@ spec: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + extraKubeletArgs: + description: ExtraKubeletArgs is a list of extra arguments to add to the kubelet. + items: + type: string + type: array + extraWriteFiles: + description: ExtraWriteFiles is a list of extra files to inject with cloud-init. + items: + description: CloudInitWriteFile is a file that will be injected by cloud-init + properties: + content: + description: Content of the file to create. + type: string + owner: + description: Owner of the file to create, e.g. "root:root" + type: string + path: + description: Path where the file should be created. + type: string + permissions: + description: Permissions of the file to create, e.g. "0600" + type: string + required: + - content + - owner + - path + - permissions + type: object + type: array httpProxy: description: The optional http proxy configuration type: string @@ -206,9 +250,24 @@ spec: type: object initConfiguration: properties: + Confinement: + description: The confinement (strict or classic) configuration + enum: + - classic + - strict + type: string IPinIP: description: The optional IPinIP configuration type: boolean + RiskLevel: + default: stable + description: The risk-level (stable, candidate, beta, or edge) for the snaps + enum: + - stable + - candidate + - beta + - edge + type: string addons: description: List of addons to be enabled upon cluster creation items: @@ -217,6 +276,35 @@ spec: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + extraKubeletArgs: + description: ExtraKubeletArgs is a list of extra arguments to add to the kubelet. + items: + type: string + type: array + extraWriteFiles: + description: ExtraWriteFiles is a list of extra files to inject with cloud-init. + items: + description: CloudInitWriteFile is a file that will be injected by cloud-init + properties: + content: + description: Content of the file to create. + type: string + owner: + description: Owner of the file to create, e.g. "root:root" + type: string + path: + description: Path where the file should be created. + type: string + permissions: + description: Permissions of the file to create, e.g. "0600" + type: string + required: + - content + - owner + - path + - permissions + type: object + type: array httpProxy: description: The optional http proxy configuration type: string diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml index c6ef732..625b10b 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml @@ -91,6 +91,12 @@ spec: schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + extraKubeletArgs: + description: ExtraKubeletArgs is a list of extra arguments to + add to the kubelet. + items: + type: string + type: array extraWriteFiles: description: ExtraWriteFiles is a list of extra files to inject with cloud-init. diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml index b9702f1..9cc6bde 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml @@ -98,6 +98,12 @@ spec: convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + extraKubeletArgs: + description: ExtraKubeletArgs is a list of extra arguments + to add to the kubelet. + items: + type: string + type: array extraWriteFiles: description: ExtraWriteFiles is a list of extra files to inject with cloud-init. From 6feeefb5def41ba4ba869e29176cd9aea27b8e0c Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 28 Nov 2022 18:46:33 +0200 Subject: [PATCH 3/3] inject extra kubelet args during cloud-init --- .../cloudinit/cloudinit_common_test.go | 70 +++++++++++++++++++ controllers/cloudinit/controlplane_init.go | 11 +++ .../cloudinit/controlplane_init_test.go | 1 + controllers/cloudinit/controlplane_join.go | 14 +++- .../cloudinit/controlplane_join_test.go | 1 + controllers/cloudinit/embed.go | 4 ++ .../cloudinit/scripts/10-configure-kubelet.sh | 26 +++++++ controllers/cloudinit/worker_join.go | 14 +++- controllers/cloudinit/worker_join_test.go | 1 + controllers/microk8sconfig_controller.go | 3 + 10 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 controllers/cloudinit/scripts/10-configure-kubelet.sh diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go index dbeacbf..1ae64d3 100644 --- a/controllers/cloudinit/cloudinit_common_test.go +++ b/controllers/cloudinit/cloudinit_common_test.go @@ -154,4 +154,74 @@ func TestCloudConfigInput(t *testing.T) { }) } }) + + t.Run("ExtraKubeletArgs", func(t *testing.T) { + for _, tc := range []struct { + name string + makeCloudConfig func(args []string) (*cloudinit.CloudConfig, error) + }{ + { + name: "ControlPlaneInit", + makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) { + return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + ExtraKubeletArgs: args, + }) + }, + }, + { + name: "ControlPlaneJoin", + makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + ExtraKubeletArgs: args, + }) + }, + }, + { + name: "Worker", + makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{ + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + ExtraKubeletArgs: args, + }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, withArgs := range []bool{true, false} { + t.Run(fmt.Sprintf("withargs=%v", withArgs), func(t *testing.T) { + g := NewWithT(t) + var args []string + if withArgs { + args = []string{"--arg=value", "--arg2=value2"} + } + c, err := tc.makeCloudConfig(args) + g.Expect(err).NotTo(HaveOccurred()) + + if withArgs { + g.Expect(c.WriteFiles).To(ContainElement(cloudinit.File{ + Content: "--arg=value\n--arg2=value2", + Path: "/var/tmp/extra-kubelet-args", + Permissions: "0400", + Owner: "root:root", + })) + } else { + for _, f := range c.WriteFiles { + g.Expect(f.Path).ToNot(Equal("/var/tmp/extra-kubelet-args")) + } + } + + _, err = cloudinit.GenerateCloudConfig(c) + g.Expect(err).NotTo(HaveOccurred()) + }) + } + }) + } + }) } diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index 367a067..3d4c8e0 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -59,6 +59,8 @@ type ControlPlaneInitInput struct { RiskLevel string // ExtraWriteFiles is a list of extra files to inject with cloud-init. ExtraWriteFiles []File + // ExtraKubeletArgs is a list of arguments to add to kubelet. + ExtraKubeletArgs []string } func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { @@ -110,11 +112,20 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { ) cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) + if args := input.ExtraKubeletArgs; len(args) > 0 { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ + Content: strings.Join(args, "\n"), + Path: filepath.Join("/var", "tmp", "extra-kubelet-args"), + Permissions: "0400", + Owner: "root:root", + }) + } cloudConfig.RunCommands = append(cloudConfig.RunCommands, "set -x", scriptPath(disableHostServicesScript), fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), + scriptPath(configureKubeletScript), "microk8s status --wait-ready", "microk8s refresh-certs /var/tmp", fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP), diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index 71fcacf..7a3274a 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -46,6 +46,7 @@ func TestControlPlaneInit(t *testing.T) { `/capi-scripts/00-disable-host-services.sh`, `/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, + `/capi-scripts/10-configure-kubelet.sh`, `microk8s status --wait-ready`, `microk8s refresh-certs /var/tmp`, `/capi-scripts/10-configure-calico-ipip.sh true`, diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 2e5cf4b..ebefbdc 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -19,6 +19,8 @@ package cloudinit import ( "fmt" "net" + "path/filepath" + "strings" "k8s.io/apimachinery/pkg/util/version" ) @@ -53,6 +55,8 @@ type ControlPlaneJoinInput struct { RiskLevel string // ExtraWriteFiles is a list of extra files to inject with cloud-init. ExtraWriteFiles []File + // ExtraKubeletArgs is a list of arguments to add to kubelet. + ExtraKubeletArgs []string } func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { @@ -84,12 +88,20 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { cloudConfig := NewBaseCloudConfig() cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) - + if args := input.ExtraKubeletArgs; len(args) > 0 { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ + Content: strings.Join(args, "\n"), + Path: filepath.Join("/var", "tmp", "extra-kubelet-args"), + Permissions: "0400", + Owner: "root:root", + }) + } cloudConfig.RunCommands = append(cloudConfig.RunCommands, "set -x", scriptPath(disableHostServicesScript), fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), + scriptPath(configureKubeletScript), "microk8s status --wait-ready", fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP), fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort), diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go index 89fb61a..b6566f9 100644 --- a/controllers/cloudinit/controlplane_join_test.go +++ b/controllers/cloudinit/controlplane_join_test.go @@ -45,6 +45,7 @@ func TestControlPlaneJoin(t *testing.T) { `/capi-scripts/00-disable-host-services.sh`, `/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, + `/capi-scripts/10-configure-kubelet.sh`, `microk8s status --wait-ready`, `/capi-scripts/10-configure-calico-ipip.sh true`, `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, diff --git a/controllers/cloudinit/embed.go b/controllers/cloudinit/embed.go index 67088cd..4636fd0 100644 --- a/controllers/cloudinit/embed.go +++ b/controllers/cloudinit/embed.go @@ -58,6 +58,9 @@ const ( // configureTraefikScript configures the control plane endpoint in the traefik provider configuration. configureTraefikScript script = "10-configure-traefik.sh" + // configureKubeletScript configures the kubelet. + configureKubeletScript script = "10-configure-kubelet.sh" + // microk8sEnableScript enables MicroK8s addons. microk8sEnableScript script = "20-microk8s-enable.sh" @@ -74,6 +77,7 @@ var allScripts = []script{ configureContainerdProxyScript, configureDqlitePortScript, configureTraefikScript, + configureKubeletScript, microk8sEnableScript, microk8sJoinScript, } diff --git a/controllers/cloudinit/scripts/10-configure-kubelet.sh b/controllers/cloudinit/scripts/10-configure-kubelet.sh new file mode 100644 index 0000000..8c243fa --- /dev/null +++ b/controllers/cloudinit/scripts/10-configure-kubelet.sh @@ -0,0 +1,26 @@ +#!/bin/bash -xe + +# Usage: +# $0 +# +# Assumptions: +# - microk8s is installed +# - /var/tmp/extra-kubelet-args exists + +EXTRA_ARGS_FILE="/var/tmp/extra-kubelet-args" +KUBELET_ARGS="/var/snap/microk8s/current/args/kubelet" + +if [ ! -f "${EXTRA_ARGS_FILE}" ]; then + echo "No extra kubelet configuration needed" + exit 0 +fi + +( + echo "" + echo "# ClusterAPI configuration" + cat "${EXTRA_ARGS_FILE}" + echo "" +) >> "${KUBELET_ARGS}" + +# restart kubelite so that kubelet picks up the new arguments +snap restart microk8s.daemon-kubelite diff --git a/controllers/cloudinit/worker_join.go b/controllers/cloudinit/worker_join.go index 96cc707..117d6dc 100644 --- a/controllers/cloudinit/worker_join.go +++ b/controllers/cloudinit/worker_join.go @@ -18,6 +18,8 @@ package cloudinit import ( "fmt" + "path/filepath" + "strings" "k8s.io/apimachinery/pkg/util/version" ) @@ -44,6 +46,8 @@ type WorkerInput struct { RiskLevel string // ExtraWriteFiles is a list of extra files to inject with cloud-init. ExtraWriteFiles []File + // ExtraKubeletArgs is a list of arguments to add to kubelet. + ExtraKubeletArgs []string } func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { @@ -66,12 +70,20 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { cloudConfig := NewBaseCloudConfig() cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) - + if args := input.ExtraKubeletArgs; len(args) > 0 { + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ + Content: strings.Join(args, "\n"), + Path: filepath.Join("/var", "tmp", "extra-kubelet-args"), + Permissions: "0400", + Owner: "root:root", + }) + } cloudConfig.RunCommands = append(cloudConfig.RunCommands, "set -x", scriptPath(disableHostServicesScript), fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), + scriptPath(configureKubeletScript), "microk8s status --wait-ready", fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort), fmt.Sprintf("%s %q --worker", scriptPath(microk8sJoinScript), fmt.Sprintf("%s:%s/%s", input.JoinNodeIP, input.ClusterAgentPort, input.Token)), diff --git a/controllers/cloudinit/worker_join_test.go b/controllers/cloudinit/worker_join_test.go index eff279e..78a2548 100644 --- a/controllers/cloudinit/worker_join_test.go +++ b/controllers/cloudinit/worker_join_test.go @@ -41,6 +41,7 @@ func TestWorkerJoin(t *testing.T) { `/capi-scripts/00-disable-host-services.sh`, `/capi-scripts/00-install-microk8s.sh "--channel 1.24 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, + `/capi-scripts/10-configure-kubelet.sh`, `microk8s status --wait-ready`, `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, `/capi-scripts/20-microk8s-join.sh "10.0.3.194:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" --worker`, diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index b274b9f..6610376 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -303,6 +303,7 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte ContainerdHTTPSProxy: microk8sConfig.Spec.InitConfiguration.HTTPSProxy, ContainerdNoProxy: microk8sConfig.Spec.InitConfiguration.NoProxy, ExtraWriteFiles: cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles), + ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, } if controlPlaneInput.TokenTTL == 0 { controlPlaneInput.TokenTTL = 315569260 @@ -396,6 +397,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con ContainerdHTTPSProxy: microk8sConfig.Spec.InitConfiguration.HTTPSProxy, ContainerdNoProxy: microk8sConfig.Spec.InitConfiguration.NoProxy, ExtraWriteFiles: cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles), + ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, } if controlPlaneInput.TokenTTL == 0 { controlPlaneInput.TokenTTL = 315569260 @@ -479,6 +481,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningWorkerNode(ctx context.Context, ClusterAgentPort: portOfNodeToConnectTo, JoinNodeIP: ipOfNodeToConnectTo, ExtraWriteFiles: cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles), + ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, } if c := microk8sConfig.Spec.InitConfiguration; c != nil {