Merge pull request #50 from canonical/MK-681/extra-kubelet-args

Support configuring extra kubelet arguments on cluster initialization
canonical · Nov 29, 2022 · d6d001e · d6d001e
2 parents 07579fa + 6feeefb
commit d6d001e
Show file tree

Hide file tree

Showing 15 changed files with 301 additions and 136 deletions.
diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go
@@ -84,6 +84,10 @@ type InitConfiguration struct {
 	// ExtraWriteFiles is a list of extra files to inject with cloud-init.
 	// +optional
 	ExtraWriteFiles []CloudInitWriteFile `json:"extraWriteFiles,omitempty"`
+
+	// ExtraKubeletArgs is a list of extra arguments to add to the kubelet.
+	// +optional
+	ExtraKubeletArgs []string `json:"extraKubeletArgs,omitempty"`
 }
 
 // CloudInitWriteFile is a file that will be injected by cloud-init

diff --git a/apis/v1beta1/zz_generated.deepcopy.go b/apis/v1beta1/zz_generated.deepcopy.go
diff --git a/bootstrap-components.yaml b/bootstrap-components.yaml
@@ -57,9 +57,24 @@ spec:
                 type: object
               initConfiguration:
                 properties:
+                  Confinement:
+                    description: The confinement (strict or classic) configuration
+                    enum:
+                    - classic
+                    - strict
+                    type: string
                   IPinIP:
                     description: The optional IPinIP configuration
                     type: boolean
+                  RiskLevel:
+                    default: stable
+                    description: The risk-level (stable, candidate, beta, or edge) for the snaps
+                    enum:
+                    - stable
+                    - candidate
+                    - beta
+                    - edge
+                    type: string
                   addons:
                     description: List of addons to be enabled upon cluster creation
                     items:
@@ -68,6 +83,35 @@ spec:
                   apiVersion:
                     description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
                     type: string
+                  extraKubeletArgs:
+                    description: ExtraKubeletArgs is a list of extra arguments to add to the kubelet.
+                    items:
+                      type: string
+                    type: array
+                  extraWriteFiles:
+                    description: ExtraWriteFiles is a list of extra files to inject with cloud-init.
+                    items:
+                      description: CloudInitWriteFile is a file that will be injected by cloud-init
+                      properties:
+                        content:
+                          description: Content of the file to create.
+                          type: string
+                        owner:
+                          description: Owner of the file to create, e.g. "root:root"
+                          type: string
+                        path:
+                          description: Path where the file should be created.
+                          type: string
+                        permissions:
+                          description: Permissions of the file to create, e.g. "0600"
+                          type: string
+                      required:
+                      - content
+                      - owner
+                      - path
+                      - permissions
+                      type: object
+                    type: array
                   httpProxy:
                     description: The optional http proxy configuration
                     type: string
@@ -206,9 +250,24 @@ spec:
                         type: object
                       initConfiguration:
                         properties:
+                          Confinement:
+                            description: The confinement (strict or classic) configuration
+                            enum:
+                            - classic
+                            - strict
+                            type: string
                           IPinIP:
                             description: The optional IPinIP configuration
                             type: boolean
+                          RiskLevel:
+                            default: stable
+                            description: The risk-level (stable, candidate, beta, or edge) for the snaps
+                            enum:
+                            - stable
+                            - candidate
+                            - beta
+                            - edge
+                            type: string
                           addons:
                             description: List of addons to be enabled upon cluster creation
                             items:
@@ -217,6 +276,35 @@ spec:
                           apiVersion:
                             description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
                             type: string
+                          extraKubeletArgs:
+                            description: ExtraKubeletArgs is a list of extra arguments to add to the kubelet.
+                            items:
+                              type: string
+                            type: array
+                          extraWriteFiles:
+                            description: ExtraWriteFiles is a list of extra files to inject with cloud-init.
+                            items:
+                              description: CloudInitWriteFile is a file that will be injected by cloud-init
+                              properties:
+                                content:
+                                  description: Content of the file to create.
+                                  type: string
+                                owner:
+                                  description: Owner of the file to create, e.g. "root:root"
+                                  type: string
+                                path:
+                                  description: Path where the file should be created.
+                                  type: string
+                                permissions:
+                                  description: Permissions of the file to create, e.g. "0600"
+                                  type: string
+                              required:
+                              - content
+                              - owner
+                              - path
+                              - permissions
+                              type: object
+                            type: array
                           httpProxy:
                             description: The optional http proxy configuration
                             type: string

diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml
@@ -91,6 +91,12 @@ spec:
                       schemas to the latest internal value, and may reject unrecognized
                       values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
                     type: string
+                  extraKubeletArgs:
+                    description: ExtraKubeletArgs is a list of extra arguments to
+                      add to the kubelet.
+                    items:
+                      type: string
+                    type: array
                   extraWriteFiles:
                     description: ExtraWriteFiles is a list of extra files to inject
                       with cloud-init.

diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml
@@ -98,6 +98,12 @@ spec:
                               convert recognized schemas to the latest internal value,
                               and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
                             type: string
+                          extraKubeletArgs:
+                            description: ExtraKubeletArgs is a list of extra arguments
+                              to add to the kubelet.
+                            items:
+                              type: string
+                            type: array
                           extraWriteFiles:
                             description: ExtraWriteFiles is a list of extra files
                               to inject with cloud-init.

diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package cloudinit_test
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -29,53 +30,65 @@ func TestCloudConfigInput(t *testing.T) {
 	t.Run("ChannelAndRiskLevel", func(t *testing.T) {
 		for _, tc := range []struct {
 			name            string
-			makeCloudConfig func() (*cloudinit.CloudConfig, error)
+			makeCloudConfig func(confinement, risk string) (*cloudinit.CloudConfig, error)
 		}{
 			{
 				name: "ControlPlaneInit",
-				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+				makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) {
 					return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
 						KubernetesVersion: "v1.25.0",
+						Confinement:       confinement,
 						Token:             strings.Repeat("a", 32),
 						TokenTTL:          100,
-						Confinement:       "strict",
-						RiskLevel:         "edge",
+						RiskLevel:         risk,
 					})
 				},
 			},
 			{
 				name: "ControlPlaneJoin",
-				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+				makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) {
 					return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
 						KubernetesVersion: "v1.25.0",
+						Confinement:       confinement,
 						Token:             strings.Repeat("a", 32),
 						TokenTTL:          100,
-						Confinement:       "strict",
-						RiskLevel:         "edge",
+						RiskLevel:         risk,
 					})
 				},
 			},
 			{
 				name: "Worker",
-				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+				makeCloudConfig: func(confinement, risk string) (*cloudinit.CloudConfig, error) {
 					return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{
 						KubernetesVersion: "v1.25.0",
+						Confinement:       confinement,
 						Token:             strings.Repeat("a", 32),
-						Confinement:       "strict",
-						RiskLevel:         "edge",
+						RiskLevel:         risk,
 					})
 				},
 			},
 		} {
 			t.Run(tc.name, func(t *testing.T) {
-				g := NewWithT(t)
-				c, err := tc.makeCloudConfig()
-				g.Expect(err).NotTo(HaveOccurred())
+				for _, confinement := range []string{"strict", "classic"} {
+					t.Run(confinement, func(t *testing.T) {
+						for _, risk := range []string{"stable", "candidate", "beta", "edge"} {
+							t.Run(risk, func(t *testing.T) {
+								g := NewWithT(t)
+								c, err := tc.makeCloudConfig(confinement, risk)
+								g.Expect(err).NotTo(HaveOccurred())
 
-				g.Expect(c.RunCommands).To(ContainElement(`/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict/edge"`))
+								if confinement == "classic" {
+									g.Expect(c.RunCommands).To(ContainElement(fmt.Sprintf(`/capi-scripts/00-install-microk8s.sh "--channel 1.25/%s --classic"`, risk)))
+								} else {
+									g.Expect(c.RunCommands).To(ContainElement(fmt.Sprintf(`/capi-scripts/00-install-microk8s.sh "--channel 1.25-strict/%s"`, risk)))
+								}
 
-				_, err = cloudinit.GenerateCloudConfig(c)
-				g.Expect(err).NotTo(HaveOccurred())
+								_, err = cloudinit.GenerateCloudConfig(c)
+								g.Expect(err).NotTo(HaveOccurred())
+							})
+						}
+					})
+				}
 			})
 		}
 	})
@@ -141,4 +154,74 @@ func TestCloudConfigInput(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("ExtraKubeletArgs", func(t *testing.T) {
+		for _, tc := range []struct {
+			name            string
+			makeCloudConfig func(args []string) (*cloudinit.CloudConfig, error)
+		}{
+			{
+				name: "ControlPlaneInit",
+				makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+						ExtraKubeletArgs:  args,
+					})
+				},
+			},
+			{
+				name: "ControlPlaneJoin",
+				makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+						ExtraKubeletArgs:  args,
+					})
+				},
+			},
+			{
+				name: "Worker",
+				makeCloudConfig: func(args []string) (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						ExtraKubeletArgs:  args,
+					})
+				},
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				for _, withArgs := range []bool{true, false} {
+					t.Run(fmt.Sprintf("withargs=%v", withArgs), func(t *testing.T) {
+						g := NewWithT(t)
+						var args []string
+						if withArgs {
+							args = []string{"--arg=value", "--arg2=value2"}
+						}
+						c, err := tc.makeCloudConfig(args)
+						g.Expect(err).NotTo(HaveOccurred())
+
+						if withArgs {
+							g.Expect(c.WriteFiles).To(ContainElement(cloudinit.File{
+								Content:     "--arg=value\n--arg2=value2",
+								Path:        "/var/tmp/extra-kubelet-args",
+								Permissions: "0400",
+								Owner:       "root:root",
+							}))
+						} else {
+							for _, f := range c.WriteFiles {
+								g.Expect(f.Path).ToNot(Equal("/var/tmp/extra-kubelet-args"))
+							}
+						}
+
+						_, err = cloudinit.GenerateCloudConfig(c)
+						g.Expect(err).NotTo(HaveOccurred())
+					})
+				}
+			})
+		}
+	})
 }
diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go
@@ -59,6 +59,8 @@ type ControlPlaneInitInput struct {
 	RiskLevel string
 	// ExtraWriteFiles is a list of extra files to inject with cloud-init.
 	ExtraWriteFiles []File
+	// ExtraKubeletArgs is a list of arguments to add to kubelet.
+	ExtraKubeletArgs []string
 }
 
 func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
@@ -110,11 +112,20 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
 	)
 	cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...)
 
+	if args := input.ExtraKubeletArgs; len(args) > 0 {
+		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{
+			Content:     strings.Join(args, "\n"),
+			Path:        filepath.Join("/var", "tmp", "extra-kubelet-args"),
+			Permissions: "0400",
+			Owner:       "root:root",
+		})
+	}
 	cloudConfig.RunCommands = append(cloudConfig.RunCommands,
 		"set -x",
 		scriptPath(disableHostServicesScript),
 		fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs),
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
+		scriptPath(configureKubeletScript),
 		"microk8s status --wait-ready",
 		"microk8s refresh-certs /var/tmp",
 		fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP),