From 5c9e6a3c6b3ffed6a21d0219989ec3a44cb23729 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Mon, 25 Nov 2024 11:34:00 +0000 Subject: [PATCH] Make cilium cni.exclusive configurable We'll add an annotation, making the cilium "cni.exclusive" setting configurable. This will allow using additional CNIs such as Multus. --- src/k8s/pkg/k8sd/features/cilium/internal.go | 7 +++ .../pkg/k8sd/features/cilium/internal_test.go | 24 +++++++++- src/k8s/pkg/k8sd/features/cilium/network.go | 5 +- .../pkg/k8sd/features/cilium/network_test.go | 46 +++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/k8s/pkg/k8sd/features/cilium/internal.go b/src/k8s/pkg/k8sd/features/cilium/internal.go index 72758019b..d55fd6195 100644 --- a/src/k8s/pkg/k8sd/features/cilium/internal.go +++ b/src/k8s/pkg/k8sd/features/cilium/internal.go @@ -21,6 +21,7 @@ type config struct { devices string directRoutingDevice string vlanBPFBypass []int + cniExclusive bool } func validateVLANBPFBypass(vlanList string) ([]int, error) { @@ -71,5 +72,11 @@ func internalConfig(annotations types.Annotations) (config, error) { c.vlanBPFBypass = vlanTags } + if v, ok := annotations.Get(apiv1_annotations.AnnotationCniExclusive); ok { + c.cniExclusive = v == "true" + } else { + c.cniExclusive = true + } + return c, nil } diff --git a/src/k8s/pkg/k8sd/features/cilium/internal_test.go b/src/k8s/pkg/k8sd/features/cilium/internal_test.go index 14af95736..ad5ce7f06 100644 --- a/src/k8s/pkg/k8sd/features/cilium/internal_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/internal_test.go @@ -21,6 +21,7 @@ func TestInternalConfig(t *testing.T) { devices: "", directRoutingDevice: "", vlanBPFBypass: nil, + cniExclusive: true, }, expectError: false, }, @@ -30,11 +31,26 @@ func TestInternalConfig(t *testing.T) { apiv1_annotations.AnnotationDevices: "eth+ lxdbr+", apiv1_annotations.AnnotationDirectRoutingDevice: "eth0", apiv1_annotations.AnnotationVLANBPFBypass: "1,2,3", + apiv1_annotations.AnnotationCniExclusive: "false", }, expectedConfig: config{ devices: "eth+ lxdbr+", directRoutingDevice: "eth0", vlanBPFBypass: []int{1, 2, 3}, + cniExclusive: false, + }, + expectError: false, + }, + { + name: "Cilum exclusive CNI", + annotations: map[string]string{ + apiv1_annotations.AnnotationCniExclusive: "true", + }, + expectedConfig: config{ + devices: "", + directRoutingDevice: "", + vlanBPFBypass: nil, + cniExclusive: true, }, expectError: false, }, @@ -45,6 +61,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{1}, + cniExclusive: true, }, expectError: false, }, @@ -55,6 +72,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{1, 2, 3, 4, 5}, + cniExclusive: true, }, expectError: false, }, @@ -65,6 +83,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{0}, + cniExclusive: true, }, expectError: false, }, @@ -96,6 +115,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{1, 2, 3}, + cniExclusive: true, }, expectError: false, }, @@ -106,6 +126,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{1, 2, 3, 4, 5}, + cniExclusive: true, }, expectError: false, }, @@ -119,7 +140,7 @@ func TestInternalConfig(t *testing.T) { { name: "Nil annotations", annotations: nil, - expectedConfig: config{}, + expectedConfig: config{cniExclusive: true}, expectError: false, }, { @@ -129,6 +150,7 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ vlanBPFBypass: []int{1, 2, 3}, + cniExclusive: true, }, expectError: false, }, diff --git a/src/k8s/pkg/k8sd/features/cilium/network.go b/src/k8s/pkg/k8sd/features/cilium/network.go index 08e4a62f7..508ec4bff 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network.go +++ b/src/k8s/pkg/k8sd/features/cilium/network.go @@ -97,8 +97,9 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, localhostAddress string, "enabled": true, }, "cni": map[string]any{ - "confPath": "/etc/cni/net.d", - "binPath": "/opt/cni/bin", + "confPath": "/etc/cni/net.d", + "binPath": "/opt/cni/bin", + "exclusive": config.cniExclusive, }, "operator": map[string]any{ "replicas": 1, diff --git a/src/k8s/pkg/k8sd/features/cilium/network_test.go b/src/k8s/pkg/k8sd/features/cilium/network_test.go index eb40caaef..86ff7d972 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/network_test.go @@ -182,6 +182,44 @@ func TestNetworkEnabled(t *testing.T) { g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateNetworkValues(g, callArgs.Values, network, snapM) }) + + t.Run("cniExclusiveDisabled", func(t *testing.T) { + g := NewWithT(t) + + helmM := &helmmock.Mock{} + snapM := &snapmock.Snap{ + Mock: snapmock.Mock{ + HelmClient: helmM, + }, + } + network := types.Network{ + Enabled: ptr.To(true), + PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), + } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } + + testAnnotations := types.Annotations{ + apiv1_annotations.AnnotationDevices: "eth+ lxdbr+", + apiv1_annotations.AnnotationDirectRoutingDevice: "eth0", + apiv1_annotations.AnnotationCniExclusive: "false", + } + status, err := ApplyNetwork(context.Background(), snapM, "127.0.0.1", apiserver, network, testAnnotations) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(status.Enabled).To(BeTrue()) + g.Expect(status.Message).To(Equal(EnabledMsg)) + g.Expect(status.Version).To(Equal(CiliumAgentImageTag)) + g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) + + callArgs := helmM.ApplyCalledWith[0] + g.Expect(callArgs.Chart).To(Equal(ChartCilium)) + g.Expect(callArgs.State).To(Equal(helm.StatePresent)) + + cniValues := callArgs.Values["cni"].(map[string]interface{}) + g.Expect(cniValues["exclusive"]).To(BeFalse()) + }) } func TestNetworkMountPath(t *testing.T) { @@ -396,4 +434,12 @@ func validateNetworkValues(g Gomega, values map[string]any, network types.Networ if exists { g.Expect(values["nodePort"].(map[string]any)["directRoutingDevice"]).To(Equal(directRoutingDevice)) } + + cniExclusiveStr, exists := annotations.Get(apiv1_annotations.AnnotationCniExclusive) + cniValues := values["cni"].(map[string]interface{}) + if exists { + g.Expect(cniValues["exclusive"]).To(Equal(cniExclusiveStr == "true")) + } else { + g.Expect(cniValues["exclusive"]).To(BeTrue()) + } }