Merge pull request #109 from canonical/eaudetcobello/KU-1189

canonical · Jul 30, 2024 · b2ccc8e · b2ccc8e
2 parents 8295d78 + 21373e2
commit b2ccc8e
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 0 deletions.
diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go
@@ -81,6 +81,10 @@ type InitConfiguration struct {
 	// +kubebuilder:default:=stable
 	RiskLevel string `json:"riskLevel,omitempty"`
 
+	// Whether or not to use the default CNI
+	// +optional
+	DisableDefaultCNI bool `json:"disableDefaultCNI,omitempty"`
+
 	// The snap store proxy domain
 	// +optional
 	SnapstoreProxyDomain string `json:"snapstoreProxyDomain,omitempty"`

diff --git a/bootstrap-components.yaml b/bootstrap-components.yaml
@@ -97,6 +97,9 @@ spec:
                     - classic
                     - strict
                     type: string
+                  disableDefaultCNI:
+                    description: Whether or not to use the default CNI
+                    type: boolean
                   extraKubeletArgs:
                     description: ExtraKubeletArgs is a list of extra arguments to
                       add to the kubelet.
@@ -369,6 +372,9 @@ spec:
                             - classic
                             - strict
                             type: string
+                          disableDefaultCNI:
+                            description: Whether or not to use the default CNI
+                            type: boolean
                           extraKubeletArgs:
                             description: ExtraKubeletArgs is a list of extra arguments
                               to add to the kubelet.

diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml
@@ -87,6 +87,9 @@ spec:
                     - classic
                     - strict
                     type: string
+                  disableDefaultCNI:
+                    description: Whether or not to use the default CNI
+                    type: boolean
                   extraKubeletArgs:
                     description: ExtraKubeletArgs is a list of extra arguments to
                       add to the kubelet.

diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml
@@ -95,6 +95,9 @@ spec:
                             - classic
                             - strict
                             type: string
+                          disableDefaultCNI:
+                            description: Whether or not to use the default CNI
+                            type: boolean
                           extraKubeletArgs:
                             description: ExtraKubeletArgs is a list of extra arguments
                               to add to the kubelet.

diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go
@@ -382,4 +382,80 @@ func TestCloudConfigInput(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("DisableDefaultCNI", func(t *testing.T) {
+		for _, tc := range []struct {
+			name            string
+			makeCloudConfig func() (*cloudinit.CloudConfig, error)
+		}{
+			{
+				name: "ControlPlaneInit",
+				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
+						DisableDefaultCNI: true,
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+					})
+				},
+			},
+			{
+				name: "ControlPlaneJoin",
+				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
+						DisableDefaultCNI: true,
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+					})
+				},
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				g := NewWithT(t)
+				c, err := tc.makeCloudConfig()
+				g.Expect(err).NotTo(HaveOccurred())
+
+				g.Expect(c.RunCommands).To(ContainElement(`/capi-scripts/10-disable-default-cni.sh`))
+			})
+		}
+	})
+
+	t.Run("DefaultCNI", func(t *testing.T) {
+		for _, tc := range []struct {
+			name            string
+			makeCloudConfig func() (*cloudinit.CloudConfig, error)
+		}{
+			{
+				name: "ControlPlaneInit",
+				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
+						DisableDefaultCNI: false,
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+					})
+				},
+			},
+			{
+				name: "ControlPlaneJoin",
+				makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
+					return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
+						DisableDefaultCNI: false,
+						KubernetesVersion: "v1.25.0",
+						Token:             strings.Repeat("a", 32),
+						TokenTTL:          100,
+					})
+				},
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				g := NewWithT(t)
+				c, err := tc.makeCloudConfig()
+				g.Expect(err).NotTo(HaveOccurred())
+
+				g.Expect(c.RunCommands).NotTo(ContainElement(`/capi-scripts/10-disable-default-cni.sh`))
+			})
+		}
+	})
 }
diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go
@@ -57,6 +57,8 @@ type ControlPlaneInitInput struct {
 	Confinement string
 	// RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels.
 	RiskLevel string
+	// DisableDefaultCNI specifies whether to disable the default CNI plugin.
+	DisableDefaultCNI bool
 	// SnapstoreProxyDomain specifies the domain of the snapstore proxy if one is to be used.
 	SnapstoreProxyDomain string
 	// SnapstoreProxyId specifies the snapstore proxy ID if one is to be used.
@@ -145,6 +147,13 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
 		scriptPath(configureKubeletScript),
 		scriptPath(waitAPIServerScript),
+	)
+
+	if input.DisableDefaultCNI {
+		cloudConfig.RunCommands = append(cloudConfig.RunCommands, scriptPath(disableDefaultCNIScript))
+	}
+
+	cloudConfig.RunCommands = append(cloudConfig.RunCommands,
 		"microk8s refresh-certs /var/tmp",
 		fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP),
 		fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),

diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go
@@ -38,6 +38,7 @@ func TestControlPlaneInit(t *testing.T) {
 			IPinIP:               true,
 			Token:                strings.Repeat("a", 32),
 			TokenTTL:             10000,
+			DisableDefaultCNI:    true,
 			Confinement:          "classic",
 		})
 		g.Expect(err).NotTo(HaveOccurred())
@@ -51,6 +52,7 @@ func TestControlPlaneInit(t *testing.T) {
 			`/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`,
 			`/capi-scripts/10-configure-kubelet.sh`,
 			`/capi-scripts/50-wait-apiserver.sh`,
+			`/capi-scripts/10-disable-default-cni.sh`,
 			`microk8s refresh-certs /var/tmp`,
 			`/capi-scripts/10-configure-calico-ipip.sh true`,
 			`/capi-scripts/10-configure-cluster-agent-port.sh "30000"`,

diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go
@@ -53,6 +53,8 @@ type ControlPlaneJoinInput struct {
 	Confinement string
 	// RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels.
 	RiskLevel string
+	// DisableDefaultCNI specifies whether to use the default CNI plugin.
+	DisableDefaultCNI bool
 	// SnapstoreProxyDomain specifies the domain of the snapstore proxy if one is to be used.
 	SnapstoreProxyDomain string
 	// SnapstoreProxyId specifies the snapstore proxy ID if one is to be used.
@@ -127,6 +129,13 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
 		fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy),
 		scriptPath(configureKubeletScript),
 		scriptPath(waitAPIServerScript),
+	)
+
+	if input.DisableDefaultCNI {
+		cloudConfig.RunCommands = append(cloudConfig.RunCommands, scriptPath(disableDefaultCNIScript))
+	}
+
+	cloudConfig.RunCommands = append(cloudConfig.RunCommands,
 		fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP),
 		fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),
 		fmt.Sprintf("%s %q", scriptPath(configureDqlitePortScript), input.DqlitePort),

diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go
@@ -34,6 +34,7 @@ func TestControlPlaneJoin(t *testing.T) {
 			ClusterAgentPort:     "30000",
 			DqlitePort:           "2379",
 			IPinIP:               true,
+			DisableDefaultCNI:    true,
 			Token:                strings.Repeat("a", 32),
 			TokenTTL:             10000,
 			JoinNodeIPs:          []string{"10.0.3.39", "10.0.3.40", "10.0.3.41"},
@@ -49,6 +50,7 @@ func TestControlPlaneJoin(t *testing.T) {
 			`/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`,
 			`/capi-scripts/10-configure-kubelet.sh`,
 			`/capi-scripts/50-wait-apiserver.sh`,
+			`/capi-scripts/10-disable-default-cni.sh`,
 			`/capi-scripts/10-configure-calico-ipip.sh true`,
 			`/capi-scripts/10-configure-cluster-agent-port.sh "30000"`,
 			`/capi-scripts/10-configure-dqlite-port.sh "2379"`,

diff --git a/controllers/cloudinit/embed.go b/controllers/cloudinit/embed.go
@@ -43,6 +43,9 @@ const (
 	// disableHostServicesScript disables services like containerd or kubelet from the host OS image.
 	disableHostServicesScript script = "00-disable-host-services.sh"
 
+	// disableDefaultCNIScript disables the default CNI plugin.
+	disableDefaultCNIScript script = "10-disable-default-cni.sh"
+
 	// installMicroK8sScript installs MicroK8s on the host.
 	installMicroK8sScript script = "00-install-microk8s.sh"
 
@@ -85,6 +88,7 @@ var allScripts = []script{
 	snapstoreHTTPProxyScript,
 	disableHostServicesScript,
 	installMicroK8sScript,
+	disableDefaultCNIScript,
 	configureCertLB,
 	configureAPIServerScript,
 	configureCalicoIPIPScript,

diff --git a/controllers/cloudinit/scripts/10-disable-default-cni.sh b/controllers/cloudinit/scripts/10-disable-default-cni.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -xe
+
+CNI_YAML="/var/snap/microk8s/current/args/cni-network/cni.yaml"
+CNI_DIR="/var/snap/microk8s/current/args/cni-network"
+
+if [ ! -f "${CNI_YAML}" ]; then
+  echo "will not disable default CNI, missing cni.yaml"
+  exit 0
+fi
+
+/capi-scripts/50-wait-apiserver.sh
+
+while ! microk8s kubectl get -f "${CNI_YAML}" > /dev/null; do
+  echo "Waiting for CNI objects to be created..."
+  sleep 5
+done
+
+microk8s kubectl delete -f "${CNI_YAML}"
+
+for file in "${CNI_DIR}"/*; do
+  mv "$file" "$file.old"
+done
diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go
@@ -314,6 +314,7 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte
 		SnapstoreProxyId:     microk8sConfig.Spec.InitConfiguration.SnapstoreProxyId,
 		Confinement:          microk8sConfig.Spec.InitConfiguration.Confinement,
 		RiskLevel:            microk8sConfig.Spec.InitConfiguration.RiskLevel,
+		DisableDefaultCNI:    microk8sConfig.Spec.InitConfiguration.DisableDefaultCNI,
 		ExtraWriteFiles:      cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles),
 		ExtraKubeletArgs:     microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs,
 		SnapstoreHTTPProxy:   microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy,
@@ -416,6 +417,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con
 		SnapstoreProxyDomain: microk8sConfig.Spec.InitConfiguration.SnapstoreProxyDomain,
 		SnapstoreProxyId:     microk8sConfig.Spec.InitConfiguration.SnapstoreProxyId,
 		RiskLevel:            microk8sConfig.Spec.InitConfiguration.RiskLevel,
+		DisableDefaultCNI:    microk8sConfig.Spec.InitConfiguration.DisableDefaultCNI,
 		Confinement:          microk8sConfig.Spec.InitConfiguration.Confinement,
 		ExtraWriteFiles:      cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles),
 		ExtraKubeletArgs:     microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs,