diff --git a/.gitignore b/.gitignore index 7203c391c8..e7f448d020 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml -test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml +test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor*.yaml # env vars file used in getting-started.md and manifests generation envvars.txt diff --git a/Makefile b/Makefile index 90b162c979..58c5d180e1 100644 --- a/Makefile +++ b/Makefile @@ -392,7 +392,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-fast-rollout.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" # for DHCP overrides @@ -410,7 +410,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" .PHONY: generate-e2e-templates-v1.10 diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index c8dfbdf730..7880cfd453 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -96,7 +96,10 @@ systemd: After=containerd.service [Service] # Make metadata environment variables available for pre-kubeadm commands. - EnvironmentFile=/run/metadata/*` + EnvironmentFile=/run/metadata/* + # Log to file + StandardOutput=append:/var/log/kubeadm-service.log + StandardError=inherit` ) func newClusterTopologyCluster(supervisorMode bool) (clusterv1.Cluster, error) { diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index 80ae91cab6..ba0c4062f7 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -261,6 +261,9 @@ spec: [Service] # Make metadata environment variables available for pre-kubeadm commands. EnvironmentFile=/run/metadata/* + # Log to file + StandardOutput=append:/var/log/kubeadm-service.log + StandardError=inherit initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -361,6 +364,9 @@ spec: [Service] # Make metadata environment variables available for pre-kubeadm commands. EnvironmentFile=/run/metadata/* + # Log to file + StandardOutput=append:/var/log/kubeadm-service.log + StandardError=inherit joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index d90f208cd7..e4541655bd 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -49,10 +49,8 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, WorkerMachineCount: ptr.To[int64](5), - // Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version - // cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30). - Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap")), - PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("fast-rollout")), + PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, } }) }) diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index df37ec751e..5fb03507da 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -169,7 +169,7 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-dhcp-overrides.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-hw-upgrade.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ignition.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-install-on-bootstrap.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-fast-rollout.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ipam.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-kcp-remediation.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-md-remediation.yaml" @@ -187,7 +187,7 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor-runtimesdk.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-fast-rollout-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" - sourcePath: "../data/shared/capv/main/metadata.yaml" @@ -267,12 +267,12 @@ providers: variables: # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml - KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.0" - KUBERNETES_VERSION: "v1.30.0" - KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.0" - KUBERNETES_VERSION_UPGRADE_TO: "v1.30.0" + KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.1" + KUBERNETES_VERSION: "v1.31.0-rc.1" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.30.0" + KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0-rc.1" KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.31" - CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.0" + CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.1" CNI: "./data/cni/calico/calico.yaml" EXP_CLUSTER_RESOURCE_SET: "true" EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml index 2ef370ebd3..49573b877c 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml @@ -12,3 +12,9 @@ patches: - target: kind: VSphereMachineTemplate path: ../commons/remove-storage-policy.yaml + - target: + kind: KubeadmControlPlane + path: ./patch-k8s-install-script-kcp.yaml + - target: + kind: KubeadmConfigTemplate + path: ./patch-k8s-install-script-kct.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml similarity index 85% rename from test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml index 6abed91df8..942726508c 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml @@ -1,15 +1,25 @@ -# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl - op: add - path: /spec/topology/variables/- + path: /spec/kubeadmConfigSpec/files/- value: - name: preKubeadmScript - # Note: This file has to be escaped as it is run through envsubst by clusterctl - value: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | #!/bin/bash set -o nounset set -o pipefail set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi function retry { attempt=0 @@ -36,8 +46,8 @@ LINE_SEPARATOR="*************************************************" echo "$${LINE_SEPARATOR}" - ## Variables (replaced by JSON patching) - KUBERNETES_VERSION=$${KUBERNETES_VERSION} + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} ## # Note: We assume if kubectl has the right version, everything else has as well @@ -55,7 +65,7 @@ CONTAINER_EXT="tar" echo "* testing CI version $${KUBERNETES_VERSION}" # Check for semver - if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" export DEBIAN_FRONTEND=noninteractive @@ -78,13 +88,16 @@ systemctl restart kubelet else CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" chmod +x "$${CI_DIR}/$${CI_PACKAGE}" - mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" done systemctl restart kubelet fi diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml new file mode 100644 index 0000000000..ab0fed842a --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml @@ -0,0 +1,125 @@ +- op: add + path: /spec/template/spec/files + value: [] +- op: add + path: /spec/template/spec/files/- + value: + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml index 4c0e41b050..4128896681 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml @@ -9,6 +9,9 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..ffb860483a --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,267 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/fast-rollout/kustomization.yaml similarity index 69% rename from test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/main/fast-rollout/kustomization.yaml index 6baeb19c74..335d205b7d 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/fast-rollout/kustomization.yaml @@ -3,9 +3,6 @@ kind: Kustomization resources: - ../topology patches: - - target: - kind: Cluster - path: ./inject-install-on-bootstrap.yaml - target: kind: Cluster path: ./set-md-rollout-strategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/set-md-rollout-strategy.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/fast-rollout/set-md-rollout-strategy.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/set-md-rollout-strategy.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/main/fast-rollout/set-md-rollout-strategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/ignition/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/ignition/kustomization.yaml index 72b598bbc4..85ecb0b7f3 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/main/ignition/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/ignition/kustomization.yaml @@ -20,3 +20,9 @@ patches: - target: kind: KubeadmConfigTemplate path: patch-user-md.yaml + - target: + kind: KubeadmControlPlane + path: ../base/patch-k8s-install-script-kcp.yaml + - target: + kind: KubeadmConfigTemplate + path: ../base/patch-k8s-install-script-kct.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml index 08f9abeccc..9907ca0906 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/kustomization.yaml @@ -6,3 +6,10 @@ resources: patchesStrategicMerge: - ../commons/cluster-resource-set-label.yaml - ../commons/cluster-network-CIDR.yaml +patches: + - target: + kind: KubeadmControlPlane + path: ./patch-k8s-install-script-kcp.yaml + - target: + kind: KubeadmConfigTemplate + path: ./patch-k8s-install-script-kct.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml similarity index 85% rename from test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml index 6abed91df8..942726508c 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kcp.yaml @@ -1,15 +1,25 @@ -# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl - op: add - path: /spec/topology/variables/- + path: /spec/kubeadmConfigSpec/files/- value: - name: preKubeadmScript - # Note: This file has to be escaped as it is run through envsubst by clusterctl - value: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | #!/bin/bash set -o nounset set -o pipefail set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi function retry { attempt=0 @@ -36,8 +46,8 @@ LINE_SEPARATOR="*************************************************" echo "$${LINE_SEPARATOR}" - ## Variables (replaced by JSON patching) - KUBERNETES_VERSION=$${KUBERNETES_VERSION} + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} ## # Note: We assume if kubectl has the right version, everything else has as well @@ -55,7 +65,7 @@ CONTAINER_EXT="tar" echo "* testing CI version $${KUBERNETES_VERSION}" # Check for semver - if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" export DEBIAN_FRONTEND=noninteractive @@ -78,13 +88,16 @@ systemctl restart kubelet else CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" chmod +x "$${CI_DIR}/$${CI_PACKAGE}" - mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" done systemctl restart kubelet fi diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml new file mode 100644 index 0000000000..ab0fed842a --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/base/patch-k8s-install-script-kct.yaml @@ -0,0 +1,125 @@ +- op: add + path: /spec/template/spec/files + value: [] +- op: add + path: /spec/template/spec/files/- + value: + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by envtest patching) + KUBERNETES_VERSION=${KUBERNETES_VERSION} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml index a0dabb0005..6858997f52 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/kustomization.yaml @@ -9,6 +9,9 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..ffb860483a --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,267 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/fast-rollout/kustomization.yaml similarity index 69% rename from test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/main/fast-rollout/kustomization.yaml index 6baeb19c74..335d205b7d 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/fast-rollout/kustomization.yaml @@ -3,9 +3,6 @@ kind: Kustomization resources: - ../topology patches: - - target: - kind: Cluster - path: ./inject-install-on-bootstrap.yaml - target: kind: Cluster path: ./set-md-rollout-strategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/set-md-rollout-strategy.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/fast-rollout/set-md-rollout-strategy.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/set-md-rollout-strategy.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/main/fast-rollout/set-md-rollout-strategy.yaml diff --git a/test/e2e/k8s_conformance_test.go b/test/e2e/k8s_conformance_test.go index 27ee4955b9..5ca2485036 100644 --- a/test/e2e/k8s_conformance_test.go +++ b/test/e2e/k8s_conformance_test.go @@ -60,10 +60,8 @@ var _ = Describe("When testing K8S conformance with K8S latest ci [supervisor] [ BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - // Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version - // cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30). - Flavor: testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap"), - PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + Flavor: testSpecificSettingsGetter().FlavorForMode("fast-rollout"), + PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, } }) }) diff --git a/test/framework/log/collector.go b/test/framework/log/collector.go index fcb540ff23..f31e248e7d 100644 --- a/test/framework/log/collector.go +++ b/test/framework/log/collector.go @@ -93,6 +93,8 @@ func (c *MachineLogCollector) CollectMachineLog(ctx context.Context, _ client.Cl "sudo", "cat", "/var/log/cloud-init.log"), captureLogs("cloud-init-output.log", "sudo", "cat", "/var/log/cloud-init-output.log"), + captureLogs("kubeadm-service.log", + "sudo", "cat", "/var/log/kubeadm-service.log"), ) }