Fix kubeadm upgrade flow for k8s version 1.30 and above (#3267)

* Fix kubeadm upgrade flow for k8s version 1.30 and above Signed-off-by: Rahul Ganesh <[email protected]> * Use semver to parse k8s version and address func signature to return pointer Signed-off-by: Rahul Ganesh <[email protected]> * update checksums --------- Signed-off-by: Rahul Ganesh <[email protected]>
aws · Jun 11, 2024 · ab5127f · ab5127f
1 parent 33851e1
commit ab5127f
Show file tree

Hide file tree

Showing 9 changed files with 581 additions and 196 deletions.
diff --git a/projects/aws/upgrader/ATTRIBUTION.txt b/projects/aws/upgrader/ATTRIBUTION.txt
@@ -1,10 +1,10 @@
 
+** github.com/aws/eks-anywhere-build-tooling/projects/aws/upgrader; version 0.0.0 --
+https://github.com/aws/eks-anywhere-build-tooling/projects/aws/upgrader
+
 ** github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/logger; version v0.0.0-20240324014530-904f2adc3ec4 --
 https://github.com/aws/eks-anywhere-build-tooling/tools/version-tracker
 
-** github.com/aws/eks-anywhere-build-tooling/upgrader; version 0.0.0 --
-https://github.com/aws/eks-anywhere-build-tooling/upgrader
-
 ** github.com/go-logr/logr; version v1.3.0 --
 https://github.com/go-logr/logr
 
@@ -289,10 +289,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ** github.com/sagikazarmark/slog-shim; version v0.1.0 --
 https://github.com/sagikazarmark/slog-shim
 
-** golang.org/go; version go1.22.3 --
+** golang.org/go; version go1.22.1 --
 https://github.com/golang/go
 
-** golang.org/x/sys/unix; version v0.18.0 --
+** golang.org/x/sys/unix; version v0.16.0 --
 https://golang.org/x/sys
 
 ** golang.org/x/text; version v0.14.0 --

diff --git a/projects/aws/upgrader/CHECKSUMS b/projects/aws/upgrader/CHECKSUMS
@@ -1,2 +1,2 @@
-87be161dbce4ba2a879d1324eaef1fb9e64360bed91d634be68419f8aa575e7d  _output/bin/upgrader/linux-amd64/upgrader
-36c982f3837aea97d7da15a0c76d065ba04e79215e379ab8a22693921792bbcf  _output/bin/upgrader/linux-arm64/upgrader
+7740e6d8f892a49c6134ba529a741b75298ac40c9d2565e5f99ca93f1e1775df  _output/bin/upgrader/linux-amd64/upgrader
+b80946b0a93fc543aa42e6b291f812f9a560779f46b5b8191007141c5d5bbb2b  _output/bin/upgrader/linux-arm64/upgrader
diff --git a/projects/aws/upgrader/go.mod b/projects/aws/upgrader/go.mod
@@ -1,6 +1,8 @@
 module github.com/aws/eks-anywhere-build-tooling/upgrader
 
-go 1.22
+go 1.22.0
+
+toolchain go1.22.4
 
 require (
 	github.com/aws/eks-anywhere-build-tooling/tools/version-tracker v0.0.0-20240324014530-904f2adc3ec4
@@ -9,31 +11,47 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
+	golang.org/x/mod v0.12.0
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/ginkgo/v2 v2.17.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
diff --git a/projects/aws/upgrader/go.sum b/projects/aws/upgrader/go.sum
diff --git a/projects/aws/upgrader/upgrade/kubeadm.go b/projects/aws/upgrader/upgrade/kubeadm.go
@@ -6,6 +6,11 @@ import (
 	"strings"
 
 	"github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/logger"
+	"golang.org/x/mod/semver"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -15,6 +20,11 @@ const (
 	yamlSeparatorWithNewLine = "---\n"
 	staticKubeVipPath        = "/etc/kubernetes/manifests/kube-vip.yaml"
 	kubeConfigPath           = "/etc/kubernetes/admin.conf"
+	upgradeConfigurationKind = "UpgradeConfiguration"
+	kubeadmAPIv1beta4        = "kubeadm.k8s.io/v1beta4"
+	kubeadmCMName            = "kubeadm-config"
+	kubeSystemNS             = "kube-system"
+	kubeVersion130           = "v1.30"
 )
 
 // KubeAdmInFirstCP upgrades the first control plane node
@@ -39,7 +49,7 @@ func (u *InPlaceUpgrader) KubeAdmInFirstCP(ctx context.Context) error {
 	kubeAdmConfigBackUp := fmt.Sprintf("%s/kubeadm-config.backup.yaml", componentsDir)
 	newKubeAdmConfig := fmt.Sprintf("%s/kubeadm-config.yaml", componentsDir)
 
-	getClusterConfigCmd := []string{"kubectl", "get", "cm", "-n", "kube-system", "kubeadm-config", "-ojsonpath='{.data.ClusterConfiguration}'", "--kubeconfig", kubeConfigPath}
+	getClusterConfigCmd := []string{"kubectl", "get", "cm", "-n", kubeSystemNS, "kubeadm-config", "-ojsonpath='{.data.ClusterConfiguration}'", "--kubeconfig", kubeConfigPath}
 	out, err := u.ExecCommand(ctx, getClusterConfigCmd[0], getClusterConfigCmd[1:]...)
 	if err != nil {
 		return execError(getClusterConfigCmd, string(out))
@@ -57,35 +67,52 @@ func (u *InPlaceUpgrader) KubeAdmInFirstCP(ctx context.Context) error {
 		}
 	}
 
-	if err = u.appendKubeletConfig(ctx, newKubeAdmConfig); err != nil {
-		return fmt.Errorf("appending kubelet config: %v", err)
-	}
-
 	if err = u.backUpAndDeleteCoreDNSConfig(ctx, componentsDir); err != nil {
 		return fmt.Errorf("backing up and deleting coreDNS config: %v", err)
 	}
 
-	kubeAdmVersionCmd := []string{"kubeadm", "version"}
+	kubeAdmVersionCmd := []string{"kubeadm", "version", "-oshort"}
 	version, err := u.ExecCommand(ctx, kubeAdmVersionCmd[0], kubeAdmVersionCmd[1:]...)
 	if err != nil {
 		return execError(kubeAdmVersionCmd, string(version))
 	}
-	logger.Info("current version of kubeadm", "cmd", "kubeadm version", "output", string(version))
+	logger.Info("current version of kubeadm", "cmd", "kubeadm version -oshort", "output", string(version))
 
-	kubeAdmUpgPlanCmd := []string{"kubeadm", "upgrade", "plan", "--ignore-preflight-errors=CoreDNSUnsupportedPlugins,CoreDNSMigration", "--config", newKubeAdmConfig}
-	kubeAdmUpgPlan, err := u.ExecCommand(ctx, kubeAdmUpgPlanCmd[0], kubeAdmUpgPlanCmd[1:]...)
-	if err != nil {
-		return execError(kubeAdmUpgPlanCmd, string(kubeAdmUpgPlan))
-	}
-	logger.Info("components to be upgraded with kubeadm", "output", string(kubeAdmUpgPlan))
+	// K8s version passed to the upgrader object is of the form vMajor.Minor.Patch-eksd-tag
+	// so it's safe to parse the version
+	kubeVersion := semver.MajorMinor(u.kubernetesVersion)
 
-	kubeAdmUpgCmd := []string{"kubeadm", "upgrade", "apply", "--ignore-preflight-errors=CoreDNSUnsupportedPlugins,CoreDNSMigration", "--config", newKubeAdmConfig, "--allow-experimental-upgrades", "--yes"}
-	kubeAdmUpg, err := u.ExecCommand(ctx, kubeAdmUpgCmd[0], kubeAdmUpgCmd[1:]...)
-	if err != nil {
-		return execError(kubeAdmUpgCmd, string(kubeAdmUpg))
-	}
-	logger.Info("verbose output for kubeadm upgrade", "output", string(kubeAdmUpg))
+	// From version 1.30 and above kubeadm upgrade needs special handling from legacy flow
+	// as --config flag starts supporting and actual kubeadm upgradeConfiguration type and not cluster configuration type
+	// Ref: https://github.com/kubernetes/kubernetes/pull/123068
+	// Issue: https://github.com/kubernetes/kubeadm/issues/3054
+	if semver.Compare(kubeVersion, kubeVersion130) >= 0 {
+		updatedClusterConfig, err := u.ReadFile(newKubeAdmConfig)
+		if err != nil {
+			return err
+		}
+		err = u.kubeAdmUpgradeVersion130AndAbove(ctx, componentsDir, string(updatedClusterConfig))
+		if err != nil {
+			return err
+		}
+	} else {
+		if err = u.appendKubeletConfig(ctx, newKubeAdmConfig); err != nil {
+			return fmt.Errorf("appending kubelet config: %v", err)
+		}
+		kubeAdmUpgPlanCmd := []string{"kubeadm", "upgrade", "plan", "--ignore-preflight-errors=CoreDNSUnsupportedPlugins,CoreDNSMigration", "--config", newKubeAdmConfig}
+		kubeAdmUpgPlan, err := u.ExecCommand(ctx, kubeAdmUpgPlanCmd[0], kubeAdmUpgPlanCmd[1:]...)
+		if err != nil {
+			return execError(kubeAdmUpgPlanCmd, string(kubeAdmUpgPlan))
+		}
+		logger.Info("components to be upgraded with kubeadm", "output", string(kubeAdmUpgPlan))
 
+		kubeAdmUpgCmd := []string{"kubeadm", "upgrade", "apply", "--ignore-preflight-errors=CoreDNSUnsupportedPlugins,CoreDNSMigration", "--config", newKubeAdmConfig, "--allow-experimental-upgrades", "--yes"}
+		kubeAdmUpg, err := u.ExecCommand(ctx, kubeAdmUpgCmd[0], kubeAdmUpgCmd[1:]...)
+		if err != nil {
+			return execError(kubeAdmUpgCmd, string(kubeAdmUpg))
+		}
+		logger.Info("verbose output for kubeadm upgrade", "output", string(kubeAdmUpg))
+	}
 	upgCmpDir, err := u.upgradeComponentsDir()
 	if err != nil {
 		return fmt.Errorf("getting upgrade components directory: %v", err)
@@ -108,6 +135,62 @@ func (u *InPlaceUpgrader) KubeAdmInFirstCP(ctx context.Context) error {
 	return nil
 }
 
+// kubeAdmUpgradeVersion130AndAbove upgrades first CP node for K8s version 1.30 & above
+//
+// As part of the upgrades:
+//  1. Update the kubeadm clusterConfig config map with latest etcd version
+//  2. Uses a kubeadm UpgradeConfiguration type for the upgrade command with apply and plan configurations
+func (u *InPlaceUpgrader) kubeAdmUpgradeVersion130AndAbove(ctx context.Context, cmpDir, clusterConfig string) error {
+	kubeAdmConfCM := generateKubeAdmConfCM(clusterConfig)
+	kubeAdmConfCMData, err := yaml.Marshal(kubeAdmConfCM)
+	if err != nil {
+		return fmt.Errorf("marshaling kubeadm-config config map: %v", err)
+	}
+
+	KubeAdmConfCMYaml := fmt.Sprintf("%s/kubeadm-config-cm.yaml", cmpDir)
+	err = u.WriteFile(KubeAdmConfCMYaml, kubeAdmConfCMData, fileMode640)
+	if err != nil {
+		return fmt.Errorf("writing kubeadm upgrade config: %v", err)
+	}
+
+	applyKubeAdmConfCMCmd := []string{"kubectl", "apply", "-f", KubeAdmConfCMYaml, "--kubeconfig", kubeConfigPath}
+	out, err := u.ExecCommand(ctx, applyKubeAdmConfCMCmd[0], applyKubeAdmConfCMCmd[1:]...)
+	if err != nil {
+		return execError(applyKubeAdmConfCMCmd, string(out))
+	}
+	logger.Info("updated config map kubeadm-config on cluster")
+
+	upgradeConfig := generateKubeAdmUpgradeConfig(u.kubernetesVersion)
+
+	upgradeConfigData, err := yaml.Marshal(&upgradeConfig)
+	if err != nil {
+		return fmt.Errorf("marshaling kubeadm upgrade config: %v", err)
+	}
+
+	kubeAdmUpgradeConfigYaml := fmt.Sprintf("%s/kubeadm-upgrade-config.yaml", cmpDir)
+	err = u.WriteFile(kubeAdmUpgradeConfigYaml, upgradeConfigData, fileMode640)
+	if err != nil {
+		return fmt.Errorf("writing kubeadm upgrade config: %v", err)
+	}
+	logger.Info("generated kubeadm upgrade config for k8s version >= 1.30", "fileLocation", kubeAdmUpgradeConfigYaml, "k8sVersion", u.kubernetesVersion)
+
+	kubeAdmUpgPlanCmd := []string{"kubeadm", "upgrade", "plan", "--config", kubeAdmUpgradeConfigYaml}
+	kubeAdmUpgPlan, err := u.ExecCommand(ctx, kubeAdmUpgPlanCmd[0], kubeAdmUpgPlanCmd[1:]...)
+	if err != nil {
+		return execError(kubeAdmUpgPlanCmd, string(kubeAdmUpgPlan))
+	}
+	logger.Info("components to be upgraded with kubeadm", "output", string(kubeAdmUpgPlan))
+
+	kubeAdmUpgCmd := []string{"kubeadm", "upgrade", "apply", "--config", kubeAdmUpgradeConfigYaml}
+	kubeAdmUpg, err := u.ExecCommand(ctx, kubeAdmUpgCmd[0], kubeAdmUpgCmd[1:]...)
+	if err != nil {
+		return execError(kubeAdmUpgCmd, string(kubeAdmUpg))
+	}
+	logger.Info("verbose output for kubeadm upgrade", "output", string(kubeAdmUpg))
+
+	return nil
+}
+
 // KubeAdmInRestCP upgrades the rest of control plane nodes
 //
 // As part of upgrade:
@@ -130,12 +213,12 @@ func (u *InPlaceUpgrader) KubeAdmInRestCP(ctx context.Context) error {
 		return fmt.Errorf("backing up and deleting coreDNS config: %v", err)
 	}
 
-	kubeAdmVersionCmd := []string{"kubeadm", "version"}
+	kubeAdmVersionCmd := []string{"kubeadm", "version", "-oshort"}
 	version, err := u.ExecCommand(ctx, kubeAdmVersionCmd[0], kubeAdmVersionCmd[1:]...)
 	if err != nil {
 		return execError(kubeAdmVersionCmd, string(version))
 	}
-	logger.Info("current version of kubeadm", "cmd", "kubeadm version", "output", string(version))
+	logger.Info("current version of kubeadm", "cmd", "kubeadm version -oshort", "output", string(version))
 
 	kubeAdmUpgNodeCmd := []string{"kubeadm", "upgrade", "node", "--ignore-preflight-errors=CoreDNSUnsupportedPlugins,CoreDNSMigration"}
 	kubeAdmUpg, err := u.ExecCommand(ctx, kubeAdmUpgNodeCmd[0], kubeAdmUpgNodeCmd[1:]...)
@@ -182,12 +265,12 @@ func (u *InPlaceUpgrader) KubeAdmInWorker(ctx context.Context) error {
 	}
 	logger.Info("Backed up and replaced kubeadm binary successfully")
 
-	kubeAdmVersionCmd := []string{"kubeadm", "version"}
+	kubeAdmVersionCmd := []string{"kubeadm", "version", "-oshort"}
 	version, err := u.ExecCommand(ctx, kubeAdmVersionCmd[0], kubeAdmVersionCmd[1:]...)
 	if err != nil {
 		return execError(kubeAdmVersionCmd, string(version))
 	}
-	logger.Info("current version of kubeadm", "cmd", "kubeadm version", "output", string(version))
+	logger.Info("current version of kubeadm", "cmd", "kubeadm version -oshort", "output", string(version))
 
 	kubeAdmUpgNodeCmd := []string{"kubeadm", "upgrade", "node"}
 	kubeAdmUpg, err := u.ExecCommand(ctx, kubeAdmUpgNodeCmd[0], kubeAdmUpgNodeCmd[1:]...)
@@ -230,7 +313,7 @@ func (u *InPlaceUpgrader) appendKubeletConfig(ctx context.Context, kubeAdmConf s
 		return err
 	}
 	conf = append(conf, []byte(yamlSeparatorWithNewLine)...)
-	getKubeletConfCmd := []string{"kubectl", "get", "cm", "-n", "kube-system", "kubelet-config", "-ojsonpath='{.data.kubelet}'", "--kubeconfig", kubeConfigPath}
+	getKubeletConfCmd := []string{"kubectl", "get", "cm", "-n", kubeSystemNS, "kubelet-config", "-ojsonpath='{.data.kubelet}'", "--kubeconfig", kubeConfigPath}
 	out, err := u.ExecCommand(ctx, getKubeletConfCmd[0], getKubeletConfCmd[1:]...)
 	if err != nil {
 		return execError(getKubeletConfCmd, string(out))
@@ -260,7 +343,7 @@ func (u *InPlaceUpgrader) appendKubeletConfig(ctx context.Context, kubeAdmConf s
 // TODO: consider using --skip-phases to skip addons/coredns once the feature flag is supported in kubeadm upgrade command
 func (u *InPlaceUpgrader) backUpAndDeleteCoreDNSConfig(ctx context.Context, cmpDir string) error {
 	coreDNSBackup := fmt.Sprintf("%s/coredns.yaml", cmpDir)
-	getCoreDNSConfCmd := []string{"kubectl", "get", "cm", "-n", "kube-system", "coredns", "-oyaml", "--kubeconfig", kubeConfigPath, "--ignore-not-found=true"}
+	getCoreDNSConfCmd := []string{"kubectl", "get", "cm", "-n", kubeSystemNS, "coredns", "-oyaml", "--kubeconfig", kubeConfigPath, "--ignore-not-found=true"}
 	coreDNSConf, err := u.ExecCommand(ctx, getCoreDNSConfCmd[0], getCoreDNSConfCmd[1:]...)
 	if err != nil {
 		return execError(getCoreDNSConfCmd, string(coreDNSConf))
@@ -272,7 +355,7 @@ func (u *InPlaceUpgrader) backUpAndDeleteCoreDNSConfig(ctx context.Context, cmpD
 			return err
 		}
 	}
-	deleteCoreDNSConfig := []string{"kubectl", "delete", "cm", "-n", "kube-system", "coredns", "--kubeconfig", kubeConfigPath, "--ignore-not-found=true"}
+	deleteCoreDNSConfig := []string{"kubectl", "delete", "cm", "-n", kubeSystemNS, "coredns", "--kubeconfig", kubeConfigPath, "--ignore-not-found=true"}
 	out, err := u.ExecCommand(ctx, deleteCoreDNSConfig[0], deleteCoreDNSConfig[1:]...)
 	if err != nil {
 		return execError(deleteCoreDNSConfig, string(out))
@@ -293,3 +376,40 @@ func (u *InPlaceUpgrader) restoreCoreDNSConfig(ctx context.Context, cmpDir strin
 	logger.Info("restored coreDNS config successfully!")
 	return nil
 }
+
+func generateKubeAdmConfCM(clusterConfig string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kubeadmCMName,
+			Namespace: kubeSystemNS,
+		},
+		Data: map[string]string{"ClusterConfiguration": clusterConfig},
+	}
+}
+
+func generateKubeAdmUpgradeConfig(version string) UpgradeConfiguration {
+	preflightErrorsList := []string{"CoreDNSUnsupportedPlugins", "CoreDNSMigration"}
+	return UpgradeConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       upgradeConfigurationKind,
+			APIVersion: kubeadmAPIv1beta4,
+		},
+		Apply: UpgradeApplyConfiguration{
+			KubernetesVersion:         version,
+			AllowExperimentalUpgrades: ptr.To(true),
+			ForceUpgrade:              ptr.To(true),
+			EtcdUpgrade:               ptr.To(true),
+			IgnorePreflightErrors:     preflightErrorsList,
+		},
+		Plan: UpgradePlanConfiguration{
+			KubernetesVersion:         version,
+			AllowExperimentalUpgrades: ptr.To(true),
+			IgnorePreflightErrors:     preflightErrorsList,
+			PrintConfig:               ptr.To(true),
+		},
+	}
+}