Make cilium cni.exclusive configurable

We'll add an annotation, making the cilium "cni.exclusive" setting
configurable. This will allow using additional CNIs such as Multus.

docs/src/snap/reference/annotations.md

@@ -142,6 +142,13 @@ Please refer to the [Kubernetes website] for more information on annnotations.
 |**Values**| \[] (string values comma separated) |
 |**Description**|Enable IP auto-detection based on which addresses on the nodes are within one of the provided CIDRs.|
 
+## `k8sd/v1alpha1/cilium/cni-exclusive`
+
+|   |   |
+|---|---|
+| **Values**| "true"\|"false"|
+| **Description**| Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. Set this to "false" if you wish to use other CNIs such as Multus. |
+
 ## `k8sd/v1alpha1/cilium/devices`
 
 |   |   |

src/k8s/pkg/k8sd/features/cilium/internal.go

@@ -21,6 +21,7 @@ type config struct {
 	devices             string
 	directRoutingDevice string
 	vlanBPFBypass       []int
+	cniExclusive        bool
 }
 
 func validateVLANBPFBypass(vlanList string) ([]int, error) {
@@ -71,5 +72,11 @@ func internalConfig(annotations types.Annotations) (config, error) {
 		c.vlanBPFBypass = vlanTags
 	}
 
+	if v, ok := annotations.Get(apiv1_annotations.AnnotationCniExclusive); ok {
+		c.cniExclusive = v == "true"
+	} else {
+		c.cniExclusive = true
+	}
+
 	return c, nil
 }

src/k8s/pkg/k8sd/features/cilium/internal_test.go

@@ -21,6 +21,7 @@ func TestInternalConfig(t *testing.T) {
 				devices:             "",
 				directRoutingDevice: "",
 				vlanBPFBypass:       nil,
+				cniExclusive:        true,
 			},
 			expectError: false,
 		},
@@ -30,11 +31,26 @@ func TestInternalConfig(t *testing.T) {
 				apiv1_annotations.AnnotationDevices:             "eth+ lxdbr+",
 				apiv1_annotations.AnnotationDirectRoutingDevice: "eth0",
 				apiv1_annotations.AnnotationVLANBPFBypass:       "1,2,3",
+				apiv1_annotations.AnnotationCniExclusive:        "false",
 			},
 			expectedConfig: config{
 				devices:             "eth+ lxdbr+",
 				directRoutingDevice: "eth0",
 				vlanBPFBypass:       []int{1, 2, 3},
+				cniExclusive:        false,
+			},
+			expectError: false,
+		},
+		{
+			name: "Cilum exclusive CNI",
+			annotations: map[string]string{
+				apiv1_annotations.AnnotationCniExclusive: "true",
+			},
+			expectedConfig: config{
+				devices:             "",
+				directRoutingDevice: "",
+				vlanBPFBypass:       nil,
+				cniExclusive:        true,
 			},
 			expectError: false,
 		},
@@ -45,6 +61,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -55,6 +72,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3, 4, 5},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -65,6 +83,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{0},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -96,6 +115,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -106,6 +126,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3, 4, 5},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -119,7 +140,7 @@ func TestInternalConfig(t *testing.T) {
 		{
 			name:           "Nil annotations",
 			annotations:    nil,
-			expectedConfig: config{},
+			expectedConfig: config{cniExclusive: true},
 			expectError:    false,
 		},
 		{
@@ -129,6 +150,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},

src/k8s/pkg/k8sd/features/cilium/network.go

@@ -97,8 +97,9 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, localhostAddress string,
 			"enabled": true,
 		},
 		"cni": map[string]any{
-			"confPath": "/etc/cni/net.d",
-			"binPath":  "/opt/cni/bin",
+			"confPath":  "/etc/cni/net.d",
+			"binPath":   "/opt/cni/bin",
+			"exclusive": config.cniExclusive,
 		},
 		"operator": map[string]any{
 			"replicas": 1,

src/k8s/pkg/k8sd/features/cilium/network_test.go

@@ -182,6 +182,44 @@ func TestNetworkEnabled(t *testing.T) {
 		g.Expect(callArgs.State).To(Equal(helm.StatePresent))
 		validateNetworkValues(g, callArgs.Values, network, snapM)
 	})
+
+	t.Run("cniExclusiveDisabled", func(t *testing.T) {
+		g := NewWithT(t)
+
+		helmM := &helmmock.Mock{}
+		snapM := &snapmock.Snap{
+			Mock: snapmock.Mock{
+				HelmClient: helmM,
+			},
+		}
+		network := types.Network{
+			Enabled: ptr.To(true),
+			PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"),
+		}
+		apiserver := types.APIServer{
+			SecurePort: ptr.To(6443),
+		}
+
+		testAnnotations := types.Annotations{
+			apiv1_annotations.AnnotationDevices:             "eth+ lxdbr+",
+			apiv1_annotations.AnnotationDirectRoutingDevice: "eth0",
+			apiv1_annotations.AnnotationCniExclusive:        "false",
+		}
+		status, err := ApplyNetwork(context.Background(), snapM, "127.0.0.1", apiserver, network, testAnnotations)
+
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(status.Enabled).To(BeTrue())
+		g.Expect(status.Message).To(Equal(EnabledMsg))
+		g.Expect(status.Version).To(Equal(CiliumAgentImageTag))
+		g.Expect(helmM.ApplyCalledWith).To(HaveLen(1))
+
+		callArgs := helmM.ApplyCalledWith[0]
+		g.Expect(callArgs.Chart).To(Equal(ChartCilium))
+		g.Expect(callArgs.State).To(Equal(helm.StatePresent))
+
+		cniValues := callArgs.Values["cni"].(map[string]interface{})
+		g.Expect(cniValues["exclusive"]).To(BeFalse())
+	})
 }
 
 func TestNetworkMountPath(t *testing.T) {
@@ -396,4 +434,12 @@ func validateNetworkValues(g Gomega, values map[string]any, network types.Networ
 	if exists {
 		g.Expect(values["nodePort"].(map[string]any)["directRoutingDevice"]).To(Equal(directRoutingDevice))
 	}
+
+	cniExclusiveStr, exists := annotations.Get(apiv1_annotations.AnnotationCniExclusive)
+	cniValues := values["cni"].(map[string]interface{})
+	if exists {
+		g.Expect(cniValues["exclusive"]).To(Equal(cniExclusiveStr == "true"))
+	} else {
+		g.Expect(cniValues["exclusive"]).To(BeTrue())
+	}
 }