Make cilium cni.exclusive configurable

We'll add an annotation, making the cilium "cni.exclusive" setting
configurable. This will allow using additional CNIs such as Multus.

docs/src/snap/reference/annotations.md

@@ -24,6 +24,7 @@ the bootstrap configuration.
 | `k8sd/v1alpha1/calico/autodetection-v6/skipInterface` | Enable IP auto-detection based on interfaces that do not match the given regex. | string |
 | `k8sd/v1alpha1/calico/autodetection-v6/canReach` | Enable IP auto-detection based on which source address on the node is used to reach the specified IP or domain. | string |
 | `k8sd/v1alpha1/calico/autodetection-v6/cidrs` | Enable IP auto-detection based on which addresses on the nodes are within one of the provided CIDRs. | []string (comma separated) |
+| `k8sd/v1alpha1/cilium/cni-exclusive` | Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. Set this to "false" if you wish to use other CNIs such as Multus. | "true"\|"false" |
 | `k8sd/v1alpha1/cilium/devices` | List of devices facing cluster/external network (used for BPF NodePort, BPF masquerading and host firewall); supports `+` as wildcard in device name, e.g. `eth+,ens+` | string |
 | `k8sd/v1alpha1/cilium/direct-routing-device` | Device name used to connect nodes in direct routing mode (used by BPF NodePort, BPF host routing); if empty, automatically set to a device with k8s InternalIP/ExternalIP or with a default route. Bridge type devices are ignored in automatic selection | string |
 | `k8sd/v1alpha1/cilium/vlan-bpf-bypass` | Comma separated list of VLAN tags to bypass eBPF filtering on native devices. Cilium enables a firewall on native devices and filters all unknown traffic, including VLAN 802.1q packets, which pass through the main device with the associated tag (e.g., VLAN device eth0.4000 and its main interface eth0). Supports `0` as wildcard for bypassing all VLANs. e.g. `4001,4002` | []string |

src/k8s/pkg/k8sd/features/cilium/internal.go

@@ -21,6 +21,7 @@ type config struct {
 	devices             string
 	directRoutingDevice string
 	vlanBPFBypass       []int
+	cniExclusive        bool
 }
 
 func validateVLANBPFBypass(vlanList string) ([]int, error) {
@@ -71,5 +72,11 @@ func internalConfig(annotations types.Annotations) (config, error) {
 		c.vlanBPFBypass = vlanTags
 	}
 
+	if v, ok := annotations.Get(apiv1_annotations.AnnotationCniExclusive); ok {
+		c.cniExclusive = v == "true"
+	} else {
+		c.cniExclusive = true
+	}
+
 	return c, nil
 }

src/k8s/pkg/k8sd/features/cilium/internal_test.go

@@ -21,6 +21,7 @@ func TestInternalConfig(t *testing.T) {
 				devices:             "",
 				directRoutingDevice: "",
 				vlanBPFBypass:       nil,
+				cniExclusive:        true,
 			},
 			expectError: false,
 		},
@@ -30,11 +31,26 @@ func TestInternalConfig(t *testing.T) {
 				apiv1_annotations.AnnotationDevices:             "eth+ lxdbr+",
 				apiv1_annotations.AnnotationDirectRoutingDevice: "eth0",
 				apiv1_annotations.AnnotationVLANBPFBypass:       "1,2,3",
+				apiv1_annotations.AnnotationCniExclusive:        "false",
 			},
 			expectedConfig: config{
 				devices:             "eth+ lxdbr+",
 				directRoutingDevice: "eth0",
 				vlanBPFBypass:       []int{1, 2, 3},
+				cniExclusive:        false,
+			},
+			expectError: false,
+		},
+		{
+			name: "Cilum exclusive CNI",
+			annotations: map[string]string{
+				apiv1_annotations.AnnotationCniExclusive: "true",
+			},
+			expectedConfig: config{
+				devices:             "",
+				directRoutingDevice: "",
+				vlanBPFBypass:       nil,
+				cniExclusive:        true,
 			},
 			expectError: false,
 		},
@@ -45,6 +61,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -55,6 +72,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3, 4, 5},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -65,6 +83,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{0},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -96,6 +115,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -106,6 +126,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3, 4, 5},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},
@@ -119,7 +140,7 @@ func TestInternalConfig(t *testing.T) {
 		{
 			name:           "Nil annotations",
 			annotations:    nil,
-			expectedConfig: config{},
+			expectedConfig: config{cniExclusive: true},
 			expectError:    false,
 		},
 		{
@@ -129,6 +150,7 @@ func TestInternalConfig(t *testing.T) {
 			},
 			expectedConfig: config{
 				vlanBPFBypass: []int{1, 2, 3},
+				cniExclusive:  true,
 			},
 			expectError: false,
 		},

src/k8s/pkg/k8sd/features/cilium/network.go

@@ -97,8 +97,9 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, localhostAddress string,
 			"enabled": true,
 		},
 		"cni": map[string]any{
-			"confPath": "/etc/cni/net.d",
-			"binPath":  "/opt/cni/bin",
+			"confPath":  "/etc/cni/net.d",
+			"binPath":   "/opt/cni/bin",
+			"exclusive": config.cniExclusive,
 		},
 		"operator": map[string]any{
 			"replicas": 1,

src/k8s/pkg/k8sd/features/cilium/network_test.go

@@ -182,6 +182,44 @@ func TestNetworkEnabled(t *testing.T) {
 		g.Expect(callArgs.State).To(Equal(helm.StatePresent))
 		validateNetworkValues(g, callArgs.Values, network, snapM)
 	})
+
+	t.Run("cniExclusiveDisabled", func(t *testing.T) {
+		g := NewWithT(t)
+
+		helmM := &helmmock.Mock{}
+		snapM := &snapmock.Snap{
+			Mock: snapmock.Mock{
+				HelmClient: helmM,
+			},
+		}
+		network := types.Network{
+			Enabled: ptr.To(true),
+			PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"),
+		}
+		apiserver := types.APIServer{
+			SecurePort: ptr.To(6443),
+		}
+
+		testAnnotations := types.Annotations{
+			apiv1_annotations.AnnotationDevices:             "eth+ lxdbr+",
+			apiv1_annotations.AnnotationDirectRoutingDevice: "eth0",
+			apiv1_annotations.AnnotationCniExclusive:        "false",
+		}
+		status, err := ApplyNetwork(context.Background(), snapM, "127.0.0.1", apiserver, network, testAnnotations)
+
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(status.Enabled).To(BeTrue())
+		g.Expect(status.Message).To(Equal(EnabledMsg))
+		g.Expect(status.Version).To(Equal(CiliumAgentImageTag))
+		g.Expect(helmM.ApplyCalledWith).To(HaveLen(1))
+
+		callArgs := helmM.ApplyCalledWith[0]
+		g.Expect(callArgs.Chart).To(Equal(ChartCilium))
+		g.Expect(callArgs.State).To(Equal(helm.StatePresent))
+
+		cniValues := callArgs.Values["cni"].(map[string]interface{})
+		g.Expect(cniValues["exclusive"]).To(BeFalse())
+	})
 }
 
 func TestNetworkMountPath(t *testing.T) {
@@ -396,4 +434,12 @@ func validateNetworkValues(g Gomega, values map[string]any, network types.Networ
 	if exists {
 		g.Expect(values["nodePort"].(map[string]any)["directRoutingDevice"]).To(Equal(directRoutingDevice))
 	}
+
+	cniExclusiveStr, exists := annotations.Get(apiv1_annotations.AnnotationCniExclusive)
+	cniValues := values["cni"].(map[string]interface{})
+	if exists {
+		g.Expect(cniValues["exclusive"]).To(Equal(cniExclusiveStr == "true"))
+	} else {
+		g.Expect(cniValues["exclusive"]).To(BeTrue())
+	}
 }