From 9a26282df3e3bda1d412366e2c3baf77cdc7ba64 Mon Sep 17 00:00:00 2001 From: Jonathan Meier Date: Mon, 26 Feb 2024 19:46:15 -0500 Subject: [PATCH] Disable ETCD Learner Mode (#7669) * Disable ETCD Learner Mode in v1.29 ETCD Learner Mode went to Beta in k8s 1.29 and is now default enabled. When a new stacked etcd instance comes up, it joins the cluster in learner mode. The API Server cannot perform rpc calls against its etcd instance, and fails to come online. In theory the etcd instance should be promoted to a full member and this issue should be resolved, but for some reason this is not happening. While we investigate a root cause, we are disabling this new feature flag explicitly. * Extract semver computation required for Kubernetes v1.29 checks --------- Co-authored-by: Abhay Krishna Arunachalam --- .../cloudstack/config/template-cp.yaml | 4 ++++ ...ted_results_encryption_config_cp_1_29.yaml | 2 ++ pkg/providers/snow/apibuilder.go | 24 ++++++++++++------- pkg/providers/snow/apibuilder_test.go | 10 +++++++- pkg/providers/snow/etcd.go | 8 +++++++ .../tinkerbell/config/template-cp.yaml | 6 ++++- pkg/providers/vsphere/config/template-cp.yaml | 4 ++++ ...esults_ubuntu_etcd_encryption_cp_1_29.yaml | 2 ++ 8 files changed, 49 insertions(+), 11 deletions(-) diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml index 8a691d3bd2e5..ed64fa9f0acd 100644 --- a/pkg/providers/cloudstack/config/template-cp.yaml +++ b/pkg/providers/cloudstack/config/template-cp.yaml @@ -67,6 +67,10 @@ spec: name: {{.controlPlaneTemplateName}} kubeadmConfigSpec: clusterConfiguration: +{{- if (ge (atoi $kube_minor_version) 29) }} + featureGates: + EtcdLearnerMode: false +{{- end }} imageRepository: {{.kubernetesRepository}} etcd: {{- if .externalEtcd }} diff --git a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml index 43c1e61c09da..f7ff2d12f60c 100644 --- a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml +++ b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml @@ -59,6 +59,8 @@ spec: name: test-control-plane-template-1234567890000 kubeadmConfigSpec: clusterConfiguration: + featureGates: + EtcdLearnerMode: false imageRepository: public.ecr.aws/eks-distro/kubernetes etcd: local: diff --git a/pkg/providers/snow/apibuilder.go b/pkg/providers/snow/apibuilder.go index 4a5773cbddbb..5e4f294f3f02 100644 --- a/pkg/providers/snow/apibuilder.go +++ b/pkg/providers/snow/apibuilder.go @@ -58,6 +58,16 @@ func KubeadmControlPlane(log logr.Logger, clusterSpec *cluster.Spec, snowMachine machineConfig := clusterSpec.SnowMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name) + kubeVersionSemver, err := semver.New(string(clusterSpec.Cluster.Spec.KubernetesVersion) + ".0") + if err != nil { + return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", clusterSpec.Cluster.Spec.KubernetesVersion, err) + } + + kube129Semver, err := semver.New(string(v1alpha1.Kube129) + ".0") + if err != nil { + return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", v1alpha1.Kube129, err) + } + osFamily := machineConfig.OSFamily() switch osFamily { case v1alpha1.Bottlerocket: @@ -70,24 +80,20 @@ func KubeadmControlPlane(log logr.Logger, clusterSpec *cluster.Spec, snowMachine addBottlerocketBootstrapSnowInKubeadmControlPlane(kcp, versionsBundle.Snow.BottlerocketBootstrapSnow) clusterapi.SetBottlerocketHostConfigInKubeadmControlPlane(kcp, machineConfig.Spec.HostOSConfiguration) + if kubeVersionSemver.Compare(kube129Semver) != -1 { + disableEtcdLearnerMode(kcp) + } + case v1alpha1.Ubuntu: kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, "/etc/eks/bootstrap.sh", ) - kubeVersionSemver, err := semver.New(string(clusterSpec.Cluster.Spec.KubernetesVersion) + ".0") - if err != nil { - return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", clusterSpec.Cluster.Spec.KubernetesVersion, err) - } - - kube129Semver, err := semver.New(string(v1alpha1.Kube129) + ".0") - if err != nil { - return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", v1alpha1.Kube129, err) - } if kubeVersionSemver.Compare(kube129Semver) != -1 { kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, "if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi", ) + disableEtcdLearnerMode(kcp) } if err := clusterapi.SetProxyConfigInKubeadmControlPlaneForUbuntu(kcp, clusterSpec.Cluster); err != nil { diff --git a/pkg/providers/snow/apibuilder_test.go b/pkg/providers/snow/apibuilder_test.go index afc86e9d4e44..654abc1a8efb 100644 --- a/pkg/providers/snow/apibuilder_test.go +++ b/pkg/providers/snow/apibuilder_test.go @@ -108,7 +108,7 @@ func wantKubeadmControlPlane(kubeVersion v1alpha1.KubernetesVersion) *controlpla wantReplicas := int32(3) wantMaxSurge := intstr.FromInt(1) versionBundles := givenVersionsBundle(kubeVersion) - return &controlplanev1.KubeadmControlPlane{ + kcp := &controlplanev1.KubeadmControlPlane{ TypeMeta: metav1.TypeMeta{ APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", Kind: "KubeadmControlPlane", @@ -200,6 +200,14 @@ func wantKubeadmControlPlane(kubeVersion v1alpha1.KubernetesVersion) *controlpla Version: versionBundles.KubeDistro.Kubernetes.Tag, }, } + + if kubeVersion == "1.29" { + kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{ + "EtcdLearnerMode": false, + } + } + + return kcp } func wantKubeadmControlPlaneUnstackedEtcd() *controlplanev1.KubeadmControlPlane { diff --git a/pkg/providers/snow/etcd.go b/pkg/providers/snow/etcd.go index 3cb43f2f11b7..b1c3d785bebe 100644 --- a/pkg/providers/snow/etcd.go +++ b/pkg/providers/snow/etcd.go @@ -15,3 +15,11 @@ func addStackedEtcdExtraArgsInKubeadmControlPlane(kcp *controlplanev1.KubeadmCon stackedEtcdExtraArgs["listen-peer-urls"] = "https://0.0.0.0:2380" stackedEtcdExtraArgs["listen-client-urls"] = "https://0.0.0.0:2379" } + +func disableEtcdLearnerMode(kcp *controlplanev1.KubeadmControlPlane) { + if kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil { + kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{} + } + + kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates["EtcdLearnerMode"] = false +} diff --git a/pkg/providers/tinkerbell/config/template-cp.yaml b/pkg/providers/tinkerbell/config/template-cp.yaml index 4d251acbe005..e82be40b8f92 100644 --- a/pkg/providers/tinkerbell/config/template-cp.yaml +++ b/pkg/providers/tinkerbell/config/template-cp.yaml @@ -1,3 +1,4 @@ +{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) -}} apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -37,6 +38,10 @@ metadata: spec: kubeadmConfigSpec: clusterConfiguration: +{{- if (ge (atoi $kube_minor_version) 29) }} + featureGates: + EtcdLearnerMode: false +{{- end }} imageRepository: {{.kubernetesRepository}} etcd: {{- if .externalEtcd }} @@ -395,7 +400,6 @@ spec: - {{ . }} {{- end }} {{- end }} -{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) }} {{- if and (or .registryMirrorMap .proxyConfig (ge (atoi $kube_minor_version) 29)) (ne .format "bottlerocket") }} preKubeadmCommands: {{- if .registryMirrorMap }} diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml index e64629fb0291..058a2bb244ae 100644 --- a/pkg/providers/vsphere/config/template-cp.yaml +++ b/pkg/providers/vsphere/config/template-cp.yaml @@ -86,6 +86,10 @@ spec: name: {{.controlPlaneTemplateName}} kubeadmConfigSpec: clusterConfiguration: +{{- if (ge (atoi $kube_minor_version) 29) }} + featureGates: + EtcdLearnerMode: false +{{- end }} imageRepository: {{.kubernetesRepository}} etcd: {{- if .externalEtcd }} diff --git a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml index dba35c17129f..dba6a5f93759 100644 --- a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml +++ b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml @@ -78,6 +78,8 @@ spec: name: test-control-plane-template-1234567890000 kubeadmConfigSpec: clusterConfiguration: + featureGates: + EtcdLearnerMode: false imageRepository: public.ecr.aws/eks-distro/kubernetes etcd: external: