Use super-admin Kubeconfig for kube-vip from Kubernetes v1.29 onwards

pkg/providers/cloudstack/config/template-cp.yaml

@@ -352,6 +352,10 @@ spec:
         else echo "{{$dir}} already symlnk";
       fi
 {{- end}}
+{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) }}
+{{- if (ge (atoi $kube_minor_version) 29) }}
+    - "if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi"
+{{- end }}
 {{- if .cloudstackControlPlaneDiskOfferingProvided }}
     diskSetup:
       filesystems:

pkg/providers/nutanix/config/cp-template.yaml

@@ -325,6 +325,10 @@ spec:
       - echo "127.0.0.1   {{`{{ ds.meta_data.hostname }}`}}" >> /etc/hosts
     postKubeadmCommands:
       - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc
+{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) }}
+{{- if (ge (atoi $kube_minor_version) 29) }}
+      - "if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi"
+{{- end }}
     useExperimentalRetryJoin: true
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

pkg/providers/snow/apibuilder.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
 	"github.com/aws/eks-anywhere/pkg/constants"
 	snowv1 "github.com/aws/eks-anywhere/pkg/providers/snow/api/v1beta1"
+	"github.com/aws/eks-anywhere/pkg/semver"
 )
 
 const (
@@ -73,6 +74,21 @@ func KubeadmControlPlane(log logr.Logger, clusterSpec *cluster.Spec, snowMachine
 		kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands,
 			"/etc/eks/bootstrap.sh",
 		)
+		kubeVersionSemver, err := semver.New(string(clusterSpec.Cluster.Spec.KubernetesVersion) + ".0")
+		if err != nil {
+			return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", clusterSpec.Cluster.Spec.KubernetesVersion, err)
+		}
+
+		kube129Semver, err := semver.New(string(v1alpha1.Kube129) + ".0")
+		if err != nil {
+			return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", v1alpha1.Kube129, err)
+		}
+
+		if kubeVersionSemver.Compare(kube129Semver) != -1 {
+			kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+				"if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi",
+			)
+		}
 
 		if err := clusterapi.SetProxyConfigInKubeadmControlPlaneForUbuntu(kcp, clusterSpec.Cluster); err != nil {
 			return nil, err

pkg/providers/snow/apibuilder_test.go

@@ -104,9 +104,10 @@ func TestCAPICluster(t *testing.T) {
 	tt.Expect(got).To(Equal(wantCAPICluster()))
 }
 
-func wantKubeadmControlPlane() *controlplanev1.KubeadmControlPlane {
+func wantKubeadmControlPlane(kubeVersion v1alpha1.KubernetesVersion) *controlplanev1.KubeadmControlPlane {
 	wantReplicas := int32(3)
 	wantMaxSurge := intstr.FromInt(1)
+	versionBundles := givenVersionsBundle(kubeVersion)
 	return &controlplanev1.KubeadmControlPlane{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "controlplane.cluster.x-k8s.io/v1beta1",
@@ -126,18 +127,18 @@ func wantKubeadmControlPlane() *controlplanev1.KubeadmControlPlane {
 			},
 			KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{
 				ClusterConfiguration: &bootstrapv1.ClusterConfiguration{
-					ImageRepository: "public.ecr.aws/eks-distro/kubernetes",
+					ImageRepository: versionBundles.KubeDistro.Kubernetes.Repository,
 					DNS: bootstrapv1.DNS{
 						ImageMeta: bootstrapv1.ImageMeta{
-							ImageRepository: "public.ecr.aws/eks-distro/coredns",
-							ImageTag:        "v1.8.4-eks-1-21-9",
+							ImageRepository: versionBundles.KubeDistro.CoreDNS.Repository,
+							ImageTag:        versionBundles.KubeDistro.CoreDNS.Tag,
 						},
 					},
 					Etcd: bootstrapv1.Etcd{
 						Local: &bootstrapv1.LocalEtcd{
 							ImageMeta: bootstrapv1.ImageMeta{
-								ImageRepository: "public.ecr.aws/eks-distro/etcd-io",
-								ImageTag:        "v3.4.16-eks-1-21-9",
+								ImageRepository: versionBundles.KubeDistro.Etcd.Repository,
+								ImageTag:        versionBundles.KubeDistro.Etcd.Tag,
 							},
 							ExtraArgs: map[string]string{
 								"cipher-suites":      "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
@@ -196,13 +197,13 @@ func wantKubeadmControlPlane() *controlplanev1.KubeadmControlPlane {
 				},
 			},
 			Replicas: &wantReplicas,
-			Version:  "v1.21.5-eks-1-21-9",
+			Version:  versionBundles.KubeDistro.Kubernetes.Tag,
 		},
 	}
 }
 
 func wantKubeadmControlPlaneUnstackedEtcd() *controlplanev1.KubeadmControlPlane {
-	kcp := wantKubeadmControlPlane()
+	kcp := wantKubeadmControlPlane("1.21")
 	kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd = bootstrapv1.Etcd{
 		External: &bootstrapv1.ExternalEtcd{
 			Endpoints: []string{},
@@ -228,7 +229,7 @@ func TestKubeadmControlPlane(t *testing.T) {
 	got, err := snow.KubeadmControlPlane(tt.logger, tt.clusterSpec, controlPlaneMachineTemplate)
 	tt.Expect(err).To(Succeed())
 
-	want := wantKubeadmControlPlane()
+	want := wantKubeadmControlPlane("1.21")
 	want.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
 	tt.Expect(got).To(BeComparableTo(want))
 }
@@ -394,7 +395,7 @@ func TestKubeadmControlPlaneWithRegistryMirrorUbuntu(t *testing.T) {
 			controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", g.machineConfigs[g.clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name], nil)
 			got, err := snow.KubeadmControlPlane(g.logger, g.clusterSpec, controlPlaneMachineTemplate)
 			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
+			want := wantKubeadmControlPlane("1.21")
 			want.Spec.KubeadmConfigSpec.Files = append(want.Spec.KubeadmConfigSpec.Files, tt.wantFiles...)
 			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(want.Spec.KubeadmConfigSpec.PreKubeadmCommands, wantRegistryMirrorCommands()...)
 			want.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
@@ -451,7 +452,7 @@ func TestKubeadmControlPlaneWithRegistryMirrorBottlerocket(t *testing.T) {
 			controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", g.machineConfigs["test-cp"], nil)
 			got, err := snow.KubeadmControlPlane(g.logger, g.clusterSpec, controlPlaneMachineTemplate)
 			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
+			want := wantKubeadmControlPlane("1.21")
 			want.Spec.KubeadmConfigSpec.Format = "bottlerocket"
 			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = []string{}
 			want.Spec.KubeadmConfigSpec.ClusterConfiguration.BottlerocketBootstrap = bootstrap
@@ -548,7 +549,7 @@ func TestKubeadmControlPlaneWithProxyConfigUbuntu(t *testing.T) {
 			controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", g.machineConfigs[g.clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name], nil)
 			got, err := snow.KubeadmControlPlane(g.logger, g.clusterSpec, controlPlaneMachineTemplate)
 			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
+			want := wantKubeadmControlPlane("1.21")
 			want.Spec.KubeadmConfigSpec.Files = append(want.Spec.KubeadmConfigSpec.Files, tt.wantFiles...)
 			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(want.Spec.KubeadmConfigSpec.PreKubeadmCommands, wantProxyConfigCommands()...)
 			want.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
@@ -557,6 +558,19 @@ func TestKubeadmControlPlaneWithProxyConfigUbuntu(t *testing.T) {
 	}
 }
 
+func TestKubeadmControlPlaneUbuntuKubernetes129(t *testing.T) {
+	tt := newApiBuilerTest(t)
+	tt.clusterSpec.Cluster.Spec.KubernetesVersion = "1.29"
+	controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", tt.machineConfigs[tt.clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name], nil)
+	got, err := snow.KubeadmControlPlane(tt.logger, tt.clusterSpec, controlPlaneMachineTemplate)
+	tt.Expect(err).To(Succeed())
+
+	want := wantKubeadmControlPlane("1.29")
+	want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(want.Spec.KubeadmConfigSpec.PreKubeadmCommands, "if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi")
+	want.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
+	tt.Expect(got).To(BeComparableTo(want))
+}
+
 func TestKubeadmControlPlaneWithProxyConfigBottlerocket(t *testing.T) {
 	for _, tt := range proxyTests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -566,7 +580,7 @@ func TestKubeadmControlPlaneWithProxyConfigBottlerocket(t *testing.T) {
 			controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", g.machineConfigs["test-cp"], nil)
 			got, err := snow.KubeadmControlPlane(g.logger, g.clusterSpec, controlPlaneMachineTemplate)
 			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
+			want := wantKubeadmControlPlane("1.21")
 			want.Spec.KubeadmConfigSpec.Format = "bottlerocket"
 			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = []string{}
 			want.Spec.KubeadmConfigSpec.ClusterConfiguration.BottlerocketBootstrap = bootstrap
@@ -679,7 +693,7 @@ func TestKubeadmControlPlaneWithBottlerocketAdditionalSettings(t *testing.T) {
 			controlPlaneMachineTemplate := snow.MachineTemplate("snow-test-control-plane-1", g.machineConfigs["test-cp"], nil)
 			got, err := snow.KubeadmControlPlane(g.logger, g.clusterSpec, controlPlaneMachineTemplate)
 			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
+			want := wantKubeadmControlPlane("1.21")
 			want.Spec.KubeadmConfigSpec.Format = "bottlerocket"
 			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = []string{}
 			want.Spec.KubeadmConfigSpec.ClusterConfiguration.BottlerocketBootstrap = bootstrap

pkg/providers/snow/objects_test.go

@@ -53,7 +53,7 @@ func TestControlPlaneObjects(t *testing.T) {
 	wantMachineTemplateName := "test-cp-2"
 	mt.SetName(wantMachineTemplateName)
 	mt.Spec.Template.Spec.InstanceType = "sbe-c.large"
-	kcp := wantKubeadmControlPlane()
+	kcp := wantKubeadmControlPlane("1.21")
 	kcp.Spec.MachineTemplate.InfrastructureRef.Name = wantMachineTemplateName
 	kcp.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
 
@@ -117,7 +117,7 @@ func TestControlPlaneObjectsWithIPPools(t *testing.T) {
 	wantMachineTemplateName := "test-cp-2"
 	mt.SetName(wantMachineTemplateName)
 	mt.Spec.Template.Spec.InstanceType = "sbe-c.large"
-	kcp := wantKubeadmControlPlane()
+	kcp := wantKubeadmControlPlane("1.21")
 	kcp.Spec.MachineTemplate.InfrastructureRef.Name = wantMachineTemplateName
 	kcp.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
 
@@ -273,7 +273,7 @@ func TestControlPlaneObjectsOldControlPlaneNotExists(t *testing.T) {
 
 	mt.SetName("snow-test-control-plane-1")
 	mt.Spec.Template.Spec.InstanceType = "sbe-c.large"
-	kcp := wantKubeadmControlPlane()
+	kcp := wantKubeadmControlPlane("1.21")
 	kcp.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
 
 	got, err := snow.ControlPlaneObjects(g.ctx, g.logger, g.clusterSpec, g.kubeconfigClient)
@@ -306,7 +306,7 @@ func TestControlPlaneObjectsOldMachineTemplateNotExists(t *testing.T) {
 
 	mt.SetName("snow-test-control-plane-1")
 	mt.Spec.Template.Spec.InstanceType = "sbe-c.large"
-	kcp := wantKubeadmControlPlane()
+	kcp := wantKubeadmControlPlane("1.21")
 	kcp.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.IgnorePreflightErrors = []string{"DirAvailable--etc-kubernetes-manifests"}
 
 	got, err := snow.ControlPlaneObjects(g.ctx, g.logger, g.clusterSpec, g.kubeconfigClient)