From 7bc27b76339edcad0ee281eeb0ab89c8b337a393 Mon Sep 17 00:00:00 2001 From: kun zhou Date: Wed, 16 Oct 2024 15:57:35 -0700 Subject: [PATCH 1/2] handle case for 1.31 cluster --- .../kubeadm/internal/cloudinit/cloudinit.go | 23 ++- .../kubeadm-bootstrap-script-pre-k8s-1-31.sh | 137 ++++++++++++++++++ .../controllers/kubeadmconfig_controller.go | 2 + 3 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go index c2f158f7c5c7..af277e200845 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go @@ -22,9 +22,11 @@ import ( "fmt" "text/template" + "github.com/blang/semver" "github.com/pkg/errors" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/util/version" ) const ( @@ -56,6 +58,7 @@ type BaseUserData struct { KubeadmCommand string KubeadmVerbosity string SentinelFileCommand string + KubernetesVersion semver.Version } func (input *BaseUserData) prepare() error { @@ -64,7 +67,7 @@ func (input *BaseUserData) prepare() error { input.KubeadmCommand = fmt.Sprintf(standardJoinCommand, input.KubeadmVerbosity) if input.UseExperimentalRetry { input.KubeadmCommand = retriableJoinScriptName - joinScriptFile, err := generateBootstrapScript(input) + joinScriptFile, err := generateBootstrapScript(input, input.KubernetesVersion) if err != nil { return errors.Wrap(err, "failed to generate user data for machine joining control plane") } @@ -118,12 +121,22 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) { } var ( + //go:embed kubeadm-bootstrap-script-pre-k8s-1-31.sh + kubeadmBootstrapScriptPre1_31 string //go:embed kubeadm-bootstrap-script.sh kubeadmBootstrapScript string + + // kubernetesVersion1_31 is the version where kubeadm removed the update-status phase + // and introduced new phases with the ControlPlaneKubeletLocalMode feature gate. + kubernetesVersion1_31 = semver.MustParse("1.31.0") ) -func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) { - joinScript, err := generate("JoinScript", kubeadmBootstrapScript, input) +func generateBootstrapScript(input interface{}, parsedversion semver.Version) (*bootstrapv1.File, error) { + bootstrapScript := kubeadmBootstrapScript + if useKubeadmBootstrapScriptPre1_31(parsedversion) { + bootstrapScript = kubeadmBootstrapScriptPre1_31 + } + joinScript, err := generate("JoinScript", bootstrapScript, input) if err != nil { return nil, errors.Wrap(err, "failed to bootstrap script for machine joins") } @@ -134,3 +147,7 @@ func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) { Content: string(joinScript), }, nil } + +func useKubeadmBootstrapScriptPre1_31(parsedversion semver.Version) bool { + return version.Compare(parsedversion, kubernetesVersion1_31, version.WithoutPreReleases()) < 0 +} diff --git a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh new file mode 100644 index 000000000000..e627bcfa54ee --- /dev/null +++ b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +log::error_exit() { + local message="${1}" + local code="${2}" + + log::error "${message}" + # {{ if .ControlPlane }} + log::info "Removing member from cluster status" + kubeadm reset -f update-cluster-status || true + log::info "Removing etcd member" + kubeadm reset -f remove-etcd-member || true + # {{ end }} + log::info "Resetting kubeadm" + kubeadm reset -f || true + log::error "cluster.x-k8s.io kubeadm bootstrap script $0 exiting with status ${code}" + exit "${code}" +} + +log::success_exit() { + log::info "cluster.x-k8s.io kubeadm bootstrap script $0 finished" + exit 0 +} + +# Log an error but keep going. +log::error() { + local message="${1}" + timestamp=$(date --iso-8601=seconds) + echo "!!! [${timestamp}] ${1}" >&2 + shift + for message; do + echo " ${message}" >&2 + done +} + +# Print a status line. Formatted to show up in a stream of output. +log::info() { + timestamp=$(date --iso-8601=seconds) + echo "+++ [${timestamp}] ${1}" + shift + for message; do + echo " ${message}" + done +} + +check_kubeadm_command() { + local command="${1}" + local code="${2}" + case ${code} in + "0") + log::info "kubeadm reported successful execution for ${command}" + ;; + "1") + log::error "kubeadm reported failed action(s) for ${command}" + ;; + "2") + log::error "kubeadm reported preflight check error during ${command}" + ;; + "3") + log::error_exit "kubeadm reported validation error for ${command}" "${code}" + ;; + *) + log::error "kubeadm reported unknown error ${code} for ${command}" + ;; + esac +} + +function retry-command() { + n=0 + local kubeadm_return + until [ $n -ge 5 ]; do + log::info "running '$*'" + # shellcheck disable=SC1083 + "$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}} + kubeadm_return=$? + check_kubeadm_command "'$*'" "${kubeadm_return}" + if [ ${kubeadm_return} -eq 0 ]; then + break + fi + # We allow preflight errors to pass + if [ ${kubeadm_return} -eq 2 ]; then + break + fi + n=$((n + 1)) + sleep 15 + done + if [ ${kubeadm_return} -ne 0 ]; then + log::error_exit "too many errors, exiting" "${kubeadm_return}" + fi +} + +# {{ if .ControlPlane }} +function try-or-die-command() { + local kubeadm_return + log::info "running '$*'" + # shellcheck disable=SC1083 + "$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}} + kubeadm_return=$? + check_kubeadm_command "'$*'" "${kubeadm_return}" + if [ ${kubeadm_return} -ne 0 ]; then + log::error_exit "fatal error, exiting" "${kubeadm_return}" + fi +} +# {{ end }} + +retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests +# {{ if .ControlPlane }} +retry-command kubeadm join phase control-plane-prepare download-certs +retry-command kubeadm join phase control-plane-prepare certs +retry-command kubeadm join phase control-plane-prepare kubeconfig +retry-command kubeadm join phase control-plane-prepare control-plane +# {{ end }} +retry-command kubeadm join phase kubelet-start +# {{ if .ControlPlane }} +try-or-die-command kubeadm join phase control-plane-join etcd +retry-command kubeadm join phase control-plane-join update-status +retry-command kubeadm join phase control-plane-join mark-control-plane +# {{ end }} + +log::success_exit \ No newline at end of file diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index d46246b453c6..16f150a3bf12 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -489,6 +489,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex Mounts: scope.Config.Spec.Mounts, DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, + KubernetesVersion: parsedVersion, }, InitConfiguration: initdata, ClusterConfiguration: clusterdata, @@ -693,6 +694,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, + KubernetesVersion: parsedVersion, }, } From c03d1a330b7fc870237a89d6b9d9e92d7aa3cffd Mon Sep 17 00:00:00 2001 From: kun zhou Date: Wed, 16 Oct 2024 22:01:22 -0700 Subject: [PATCH 2/2] modify bootstrap script for 1.31 cluster --- .../internal/cloudinit/kubeadm-bootstrap-script.sh | 12 ++++-------- .../internal/controllers/kubeadmconfig_controller.go | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh index 5c78238ce0ac..4a3aa5d0d860 100644 --- a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh +++ b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 The Kubernetes Authors. +# Copyright 2024 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -106,7 +106,6 @@ function retry-command() { fi } -# {{ if .ControlPlane }} function try-or-die-command() { local kubeadm_return log::info "running '$*'" @@ -118,7 +117,6 @@ function try-or-die-command() { log::error_exit "fatal error, exiting" "${kubeadm_return}" fi } -# {{ end }} retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests # {{ if .ControlPlane }} @@ -128,10 +126,8 @@ retry-command kubeadm join phase control-plane-prepare kubeconfig retry-command kubeadm join phase control-plane-prepare control-plane # {{ end }} retry-command kubeadm join phase kubelet-start -# {{ if .ControlPlane }} -try-or-die-command kubeadm join phase control-plane-join etcd -retry-command kubeadm join phase control-plane-join update-status -retry-command kubeadm join phase control-plane-join mark-control-plane -# {{ end }} + +# Run kubeadm join and skip all already executed phases. +try-or-die-command kubeadm join --skip-phases preflight,control-plane-prepare,kubelet-start log::success_exit diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 16f150a3bf12..49387855ecd6 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -590,6 +590,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, + KubernetesVersion: parsedVersion, }, JoinConfiguration: joinData, }