From bfa57a27e6a1b5941fdea07e1945b2bd75a45c9e Mon Sep 17 00:00:00 2001 From: Ilja Rotar <77339620+iljarotar@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:03:27 +0200 Subject: [PATCH] Update Cilium extension to support Cilium 1.15 (#403) --- .../templates/cloud-controller-manager.yaml | 2 ++ charts/internal/control-plane/values.yaml | 1 + .../templates/metallb-crds.yaml | 2 ++ .../templates/metallb.yaml | 2 ++ .../templates/rbac-node-controller.yaml | 29 ++++++++++++++++++- .../internal/shoot-control-plane/values.yaml | 6 ++++ go.mod | 2 +- go.sum | 4 +-- pkg/admission/mutator/config.go | 10 ++++++- pkg/admission/mutator/defaulter.go | 10 +++++++ pkg/admission/mutator/defaulter_test.go | 4 ++- pkg/controller/controlplane/valuesprovider.go | 19 +++++++++++- pkg/controller/healthcheck/add.go | 5 ++++ pkg/controller/worker/actuator.go | 7 +++-- 14 files changed, 93 insertions(+), 10 deletions(-) diff --git a/charts/internal/control-plane/templates/cloud-controller-manager.yaml b/charts/internal/control-plane/templates/cloud-controller-manager.yaml index b4723300f..192c2ee07 100644 --- a/charts/internal/control-plane/templates/cloud-controller-manager.yaml +++ b/charts/internal/control-plane/templates/cloud-controller-manager.yaml @@ -105,6 +105,8 @@ spec: value: {{ .Values.cloudControllerManager.additionalNetworks }} - name: METAL_SSH_PUBLICKEY value: {{ .Values.cloudControllerManager.sshPublicKey | quote }} + - name: LOADBALANCER + value: {{ .Values.cloudControllerManager.loadBalancer }} livenessProbe: httpGet: path: /healthz diff --git a/charts/internal/control-plane/values.yaml b/charts/internal/control-plane/values.yaml index 357d35ec4..ffdda1914 100644 --- a/charts/internal/control-plane/values.yaml +++ b/charts/internal/control-plane/values.yaml @@ -37,6 +37,7 @@ cloudControllerManager: clusterID: cluster-id defaultExternalNetwork: external-network-id additionalNetworks: internet,mpls + loadBalancer: metallb sshPublicKey: publickey metal: endpoint: api-url diff --git a/charts/internal/shoot-control-plane/templates/metallb-crds.yaml b/charts/internal/shoot-control-plane/templates/metallb-crds.yaml index 8eeaee929..f47f13838 100644 --- a/charts/internal/shoot-control-plane/templates/metallb-crds.yaml +++ b/charts/internal/shoot-control-plane/templates/metallb-crds.yaml @@ -1,3 +1,4 @@ +{{- if .Values.metallb.enabled }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1057,3 +1058,4 @@ spec: storage: true subresources: status: {} +{{- end }} diff --git a/charts/internal/shoot-control-plane/templates/metallb.yaml b/charts/internal/shoot-control-plane/templates/metallb.yaml index d4790333a..c198200e3 100644 --- a/charts/internal/shoot-control-plane/templates/metallb.yaml +++ b/charts/internal/shoot-control-plane/templates/metallb.yaml @@ -1,3 +1,4 @@ +{{- if .Values.metallb.enabled }} apiVersion: v1 kind: Namespace metadata: @@ -671,3 +672,4 @@ roleRef: subjects: - kind: ServiceAccount name: controller +{{- end }} diff --git a/charts/internal/shoot-control-plane/templates/rbac-node-controller.yaml b/charts/internal/shoot-control-plane/templates/rbac-node-controller.yaml index 8088c6108..c585ef61a 100644 --- a/charts/internal/shoot-control-plane/templates/rbac-node-controller.yaml +++ b/charts/internal/shoot-control-plane/templates/rbac-node-controller.yaml @@ -139,6 +139,33 @@ rules: - get - create - update +{{- if .Values.cilium.enabled }} +- apiGroups: + - cilium.io + resources: + - ciliumbgppeeringpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end }} +{{- if .Values.metallb.enabled }} - apiGroups: - metallb.io resources: @@ -156,7 +183,6 @@ rules: resources: - ipaddresspools verbs: - - create - create - delete - get @@ -176,6 +202,7 @@ rules: - patch - update - watch +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/charts/internal/shoot-control-plane/values.yaml b/charts/internal/shoot-control-plane/values.yaml index b832fb8dd..6d1e13aed 100644 --- a/charts/internal/shoot-control-plane/values.yaml +++ b/charts/internal/shoot-control-plane/values.yaml @@ -16,6 +16,12 @@ duros: enabled: false endpoints: [] +cilium: + enabled: false + +metallb: + enabled: true + nodeInit: enabled: true diff --git a/go.mod b/go.mod index 582a07edc..7d5462ded 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gardener/etcd-druid v0.22.0 github.com/gardener/gardener v1.96.6 github.com/gardener/gardener-extension-networking-calico v1.39.1 - github.com/gardener/gardener-extension-networking-cilium v1.34.0 + github.com/gardener/gardener-extension-networking-cilium v1.35.0 github.com/gardener/machine-controller-manager v0.53.0 github.com/go-logr/logr v1.4.2 github.com/go-openapi/strfmt v0.23.0 diff --git a/go.sum b/go.sum index 5eea070a0..82e654c99 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,8 @@ github.com/gardener/gardener v1.96.6 h1:SWiK4U8UdxIb9GvN9XcZa1GIQEP+Ul5wAlgl5st0 github.com/gardener/gardener v1.96.6/go.mod h1:wXAk6DzltkuJzWvAmIvw1/GscfCn2Po3LWWCr4oCbiQ= github.com/gardener/gardener-extension-networking-calico v1.39.1 h1:x/PeBSXTasyeSHY6Q0czp9mhCsF0N1FHsH3j0/EEqMc= github.com/gardener/gardener-extension-networking-calico v1.39.1/go.mod h1:02QjW3PPk4gzGZAcKiEMBtUOfBw+6rPgYt4ZGRkbJbY= -github.com/gardener/gardener-extension-networking-cilium v1.34.0 h1:GpTNR6Ra+J8jv77S9GCh+JnpA+MN2/42TbkcZWcOB7Y= -github.com/gardener/gardener-extension-networking-cilium v1.34.0/go.mod h1:2b2ED5C7Nch4r772YKRDEpC/9Ak8SJB9gblDqBfysYM= +github.com/gardener/gardener-extension-networking-cilium v1.35.0 h1:yKkiOcs1YybHEiExR2tOLD5lF3c96fS6RrNvit1xdM8= +github.com/gardener/gardener-extension-networking-cilium v1.35.0/go.mod h1:zMsv8Hv+MSr3R/OQ0a+fJesygCXJNuIUPmcRol/R4W8= github.com/gardener/hvpa-controller/api v0.15.0 h1:igsalL5Z6kFMn1+Kv1Eq0cRjYW+4oBA1aEY/yDO2QtI= github.com/gardener/hvpa-controller/api v0.15.0/go.mod h1:fqb4wNrQLESDKpm7ppXyCM2Gvx96wRlLL35aH0ge07U= github.com/gardener/machine-controller-manager v0.53.0 h1:g2O0F7nEYZ9LjyPY6Gew8+q0n+rU88deexNq5k8CKks= diff --git a/pkg/admission/mutator/config.go b/pkg/admission/mutator/config.go index 070cef9cc..0d6aa6b43 100644 --- a/pkg/admission/mutator/config.go +++ b/pkg/admission/mutator/config.go @@ -60,7 +60,15 @@ func (c *config) ciliumTunnel() ciliumextensionv1alpha1.TunnelMode { } func (c *config) ciliumDevices() []string { - return c.slice("DEFAULTER_CILIUMDEVICES", []string{"lan+"}) + return c.slice("DEFAULTER_CILIUMDEVICES", []string{"lan+", "lo"}) +} + +func (c *config) ciliumDirectRoutingDevice() string { + return c.string("DEFAULTER_CILIUMDIRECTROUTINGDEVICE", "lo") +} + +func (c *config) bgpControlPlaneEnabled() bool { + return c.bool("DEFAULTER_CILIUMBGPCONTROLPLANE", true) } func (c *config) ciliumIPv4NativeRoutingCIDREnabled() bool { diff --git a/pkg/admission/mutator/defaulter.go b/pkg/admission/mutator/defaulter.go index 95559498b..8bf43762d 100644 --- a/pkg/admission/mutator/defaulter.go +++ b/pkg/admission/mutator/defaulter.go @@ -190,6 +190,16 @@ func (d *defaulter) defaultCiliumConfig(shoot *gardenv1beta1.Shoot) error { networkConfig.Devices = d.c.ciliumDevices() } + if networkConfig.DirectRoutingDevice == nil { + networkConfig.DirectRoutingDevice = pointer.Pointer(d.c.ciliumDirectRoutingDevice()) + } + + if networkConfig.BGPControlPlane == nil { + networkConfig.BGPControlPlane = &ciliumextensionv1alpha1.BGPControlPlane{ + Enabled: d.c.bgpControlPlaneEnabled(), + } + } + if networkConfig.IPv4NativeRoutingCIDREnabled == nil { networkConfig.IPv4NativeRoutingCIDREnabled = pointer.Pointer(d.c.ciliumIPv4NativeRoutingCIDREnabled()) } diff --git a/pkg/admission/mutator/defaulter_test.go b/pkg/admission/mutator/defaulter_test.go index 2518afd47..f7fd2cb73 100644 --- a/pkg/admission/mutator/defaulter_test.go +++ b/pkg/admission/mutator/defaulter_test.go @@ -342,9 +342,11 @@ func Test_defaulter_defaultShoot(t *testing.T) { }, TunnelMode: pointer.Pointer(ciliumextensionv1alpha1.Disabled), MTU: pointer.Pointer(1440), - Devices: []string{"lan+"}, + Devices: []string{"lan+", "lo"}, + DirectRoutingDevice: pointer.Pointer("lo"), LoadBalancingMode: pointer.Pointer(ciliumextensionv1alpha1.DSR), IPv4NativeRoutingCIDREnabled: pointer.Pointer(true), + BGPControlPlane: &ciliumextensionv1alpha1.BGPControlPlane{Enabled: true}, }, }, }, diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index f319410d9..06c1d6674 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -29,9 +29,10 @@ import ( apismetal "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal" "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/helper" - metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client" metalgo "github.com/metal-stack/metal-go" + metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -488,10 +489,18 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, c "enabled": vp.controllerConfig.Storage.Duros.Enabled, } + ciliumValues := map[string]any{ + "enabled": false, + } + metallbValues := map[string]any{ + "enabled": true, + } nodeInitValues := map[string]any{ "enabled": true, } if pointer.SafeDeref(pointer.SafeDeref(cluster.Shoot.Spec.Networking).Type) == "cilium" { + ciliumValues["enabled"] = true + metallbValues["enabled"] = false nodeInitValues["enabled"] = false } @@ -608,6 +617,8 @@ func (vp *valuesProvider) getControlPlaneShootChartValues(ctx context.Context, c "apiserverIPs": apiserverIPs, "nodeCIDR": nodeCIDR, "duros": durosValues, + "cilium": ciliumValues, + "metallb": metallbValues, "nodeInit": nodeInitValues, "restrictEgress": map[string]any{ // FIXME remove "enabled": cpConfig.FeatureGates.RestrictEgress != nil && *cpConfig.FeatureGates.RestrictEgress, @@ -733,6 +744,11 @@ func getCCMChartValues( return nil, fmt.Errorf("secret %q not found", metal.CloudControllerManagerServerName) } + loadBalancer := "metallb" + if pointer.SafeDeref(cluster.Shoot.Spec.Networking.Type) == "cilium" { + loadBalancer = "cilium" + } + values := map[string]interface{}{ "cloudControllerManager": map[string]interface{}{ "replicas": extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1), @@ -743,6 +759,7 @@ func getCCMChartValues( "podNetwork": extensionscontroller.GetPodNetwork(cluster), "defaultExternalNetwork": defaultExternalNetwork, "additionalNetworks": strings.Join(infrastructureConfig.Firewall.Networks, ","), + "loadBalancer": loadBalancer, "sshPublicKey": string(sshSecret.Data["id_rsa.pub"]), "metal": map[string]interface{}{ "endpoint": mcp.Endpoint, diff --git a/pkg/controller/healthcheck/add.go b/pkg/controller/healthcheck/add.go index 920501a8d..44e8521bd 100644 --- a/pkg/controller/healthcheck/add.go +++ b/pkg/controller/healthcheck/add.go @@ -7,6 +7,7 @@ import ( healthcheckconfig "github.com/gardener/gardener/extensions/pkg/apis/config" "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/config" "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal" + "github.com/metal-stack/metal-lib/pkg/pointer" extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" genericcontrolplaneactuator "github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator" @@ -47,6 +48,9 @@ func RegisterHealthChecks(ctx context.Context, mgr manager.Manager, opts AddOpti durosPreCheck := func(_ context.Context, _ client.Client, _ client.Object, _ *extensionscontroller.Cluster) bool { return opts.ControllerConfig.Storage.Duros.Enabled } + metallbPreCheck := func(_ context.Context, _ client.Client, _ client.Object, cluster *extensionscontroller.Cluster) bool { + return pointer.SafeDeref(cluster.Shoot.Spec.Networking.Type) == "calico" + } if err := healthcheck.DefaultRegistration( ctx, @@ -82,6 +86,7 @@ func RegisterHealthChecks(ctx context.Context, mgr manager.Manager, opts AddOpti { ConditionType: string(gardencorev1beta1.ShootSystemComponentsHealthy), HealthCheck: CheckMetalLB(), + PreCheckFunc: metallbPreCheck, }, }, // TODO(acumino): Remove this condition in a future release. diff --git a/pkg/controller/worker/actuator.go b/pkg/controller/worker/actuator.go index 646729363..c8001fb5b 100644 --- a/pkg/controller/worker/actuator.go +++ b/pkg/controller/worker/actuator.go @@ -11,13 +11,14 @@ import ( "github.com/gardener/gardener/extensions/pkg/controller/worker" "github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/config" - apismetal "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal" - metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client" metalgo "github.com/metal-stack/metal-go" "github.com/metal-stack/metal-go/api/models" "github.com/metal-stack/metal-lib/pkg/cache" + "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/config" + apismetal "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal" + metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" gardener "github.com/gardener/gardener/pkg/client/kubernetes"