Support ControlPlaneKubeletLocalMode feature gate for k8sVersion >= 1…

….31.0 (#218) Co-authored-by: Kun Zhou <[email protected]> Co-authored-by: Snehal Amrutkar <[email protected]>
spectrocloud · Nov 13, 2024 · 35c21dd · 35c21dd
1 parent 18c3163
commit 35c21dd
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 0 deletions.
diff --git a/controlplane/kubeadm/internal/controllers/scale.go b/controlplane/kubeadm/internal/controllers/scale.go
@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"sigs.k8s.io/cluster-api/util/version"
 	"strings"
 
 	"github.com/blang/semver"
@@ -54,6 +55,13 @@ func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Conte
 	}
 
 	bootstrapSpec := controlPlane.InitialControlPlaneConfig()
+	// We intentionally only parse major/minor/patch so that the subsequent code
+	// also already applies to beta versions of new releases.
+	parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version)
+	}
+	internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant)
 	fd := controlPlane.NextFailureDomainForScaleUp()
 	if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil {
 		logger.Error(err, "Failed to create initial control plane Machine")
@@ -75,6 +83,13 @@ func (r *KubeadmControlPlaneReconciler) scaleUpControlPlane(ctx context.Context,
 
 	// Create the bootstrap configuration
 	bootstrapSpec := controlPlane.JoinControlPlaneConfig()
+	// We intentionally only parse major/minor/patch so that the subsequent code
+	// also already applies to beta versions of new releases.
+	parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version)
+	}
+	internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant)
 	fd := controlPlane.NextFailureDomainForScaleUp()
 	if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil {
 		logger.Error(err, "Failed to create additional control plane Machine")

diff --git a/controlplane/kubeadm/internal/controllers/upgrade.go b/controlplane/kubeadm/internal/controllers/upgrade.go
@@ -88,6 +88,10 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
 		if err := workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(ctx, imageRepository, parsedVersion); err != nil {
 			return ctrl.Result{}, errors.Wrap(err, "failed to update the image repository in the kubeadm config map")
 		}
+
+		if err := workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec, parsedVersionTolerant); err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to update feature gates in the kubeadm config map")
+		}
 	}
 
 	if kcp.Spec.KubeadmConfigSpec.ClusterConfiguration != nil && kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local != nil {

diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go
@@ -89,6 +89,13 @@ var (
 	// ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes
 	// to remove an etcd member.
 	ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported")
+
+	// minKubernetesVersionControlPlaneKubeletLocalMode is the min version from which
+	// we will enable the ControlPlaneKubeletLocalMode kubeadm feature gate.
+	// Note: We have to do this with Kubernetes 1.31. Because with that version we encountered
+	// a case where it's not okay anymore to ignore the Kubernetes version skew (kubelet 1.31 uses
+	// the spec.clusterIP field selector that is only implemented in kube-apiserver >= 1.31.0).
+	minKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.31.0")
 )
 
 // WorkloadCluster defines all behaviors necessary to upgrade kubernetes on a workload cluster
@@ -107,6 +114,7 @@ type WorkloadCluster interface {
 	ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error
 	UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error
 	UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error
+	UpdateFeatureGatesInKubeadmConfigMap(ctx context.Context, kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) error
 	UpdateEtcdVersionInKubeadmConfigMap(ctx context.Context, imageRepository, imageTag string, version semver.Version) error
 	UpdateEtcdExtraArgsInKubeadmConfigMap(ctx context.Context, extraArgs map[string]string, version semver.Version) error
 	UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error
@@ -181,6 +189,44 @@ func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context,
 	}, version)
 }
 
+// UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map.
+func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(ctx context.Context, kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) error {
+	return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
+		// We use DeepCopy here to avoid modifying the KCP object in the apiserver.
+		kubeadmConfigSpec := kubeadmConfigSpec.DeepCopy()
+		DefaultFeatureGates(kubeadmConfigSpec, kubernetesVersion)
+
+		// Even if featureGates is nil, reset it to ClusterConfiguration
+		// to override any previously set feature gates.
+		c.FeatureGates = kubeadmConfigSpec.ClusterConfiguration.FeatureGates
+	}, kubernetesVersion)
+}
+
+const (
+	// ControlPlaneKubeletLocalMode is a feature gate of kubeadm that ensures
+	// kubelets only communicate with the local apiserver.
+	ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode"
+)
+
+// DefaultFeatureGates defaults the feature gates field.
+func DefaultFeatureGates(kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) {
+	if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) {
+		return
+	}
+
+	if kubeadmConfigSpec.ClusterConfiguration == nil {
+		kubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
+	}
+
+	if kubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil {
+		kubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{}
+	}
+
+	if _, ok := kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode]; !ok {
+		kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode] = true
+	}
+}
+
 // UpdateKubernetesVersionInKubeadmConfigMap updates the kubernetes version in the kubeadm config map.
 func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error {
 	return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {