Skip to content

Commit

Permalink
bump Kubernetes version in tests to 1.31.0-rc.0
Browse files Browse the repository at this point in the history
  • Loading branch information
chrischdi committed Aug 7, 2024
1 parent bb14455 commit 169889a
Show file tree
Hide file tree
Showing 23 changed files with 879 additions and 43 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml
test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml
test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor*.yaml

# env vars file used in getting-started.md and manifests generation
envvars.txt
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-fast-rollout.yaml"
# for PCI passthrough template
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
# for DHCP overrides
Expand All @@ -410,7 +410,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"

.PHONY: generate-e2e-templates-v1.10
Expand Down
5 changes: 4 additions & 1 deletion packaging/flavorgen/flavors/generators.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,10 @@ systemd:
After=containerd.service
[Service]
# Make metadata environment variables available for pre-kubeadm commands.
EnvironmentFile=/run/metadata/*`
EnvironmentFile=/run/metadata/*
# Log to file
StandardOutput=append:/var/log/kubeadm-service.log
StandardError=inherit`
)

func newClusterTopologyCluster(supervisorMode bool) (clusterv1.Cluster, error) {
Expand Down
6 changes: 6 additions & 0 deletions templates/cluster-template-ignition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ spec:
[Service]
# Make metadata environment variables available for pre-kubeadm commands.
EnvironmentFile=/run/metadata/*
# Log to file
StandardOutput=append:/var/log/kubeadm-service.log
StandardError=inherit
initConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
Expand Down Expand Up @@ -361,6 +364,9 @@ spec:
[Service]
# Make metadata environment variables available for pre-kubeadm commands.
EnvironmentFile=/run/metadata/*
# Log to file
StandardOutput=append:/var/log/kubeadm-service.log
StandardError=inherit
joinConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
Expand Down
6 changes: 2 additions & 4 deletions test/e2e/cluster_upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,8 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
WorkerMachineCount: ptr.To[int64](5),
// Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version
// cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30).
Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap")),
PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("fast-rollout")),
PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
}
})
})
Expand Down
14 changes: 7 additions & 7 deletions test/e2e/config/vsphere.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ providers:
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-dhcp-overrides.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-hw-upgrade.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ignition.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-install-on-bootstrap.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-fast-rollout.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ipam.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-kcp-remediation.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-md-remediation.yaml"
Expand All @@ -187,7 +187,7 @@ providers:
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor-runtimesdk.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-fast-rollout-supervisor.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml"
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
- sourcePath: "../data/shared/capv/main/metadata.yaml"
Expand Down Expand Up @@ -267,12 +267,12 @@ providers:

variables:
# Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.0"
KUBERNETES_VERSION: "v1.30.0"
KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.0"
KUBERNETES_VERSION_UPGRADE_TO: "v1.30.0"
KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.1"
KUBERNETES_VERSION: "v1.31.0-rc.1"
KUBERNETES_VERSION_UPGRADE_FROM: "v1.30.0"
KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0-rc.1"
KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.31"
CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.0"
CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.1"
CNI: "./data/cni/calico/calico.yaml"
EXP_CLUSTER_RESOURCE_SET: "true"
EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,9 @@ patches:
- target:
kind: VSphereMachineTemplate
path: ../commons/remove-storage-policy.yaml
- target:
kind: KubeadmControlPlane
path: ./patch-k8s-install-script-kcp.yaml
- target:
kind: KubeadmConfigTemplate
path: ./patch-k8s-install-script-kct.yaml
Original file line number Diff line number Diff line change
@@ -1,15 +1,25 @@
# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl
- op: add
path: /spec/topology/variables/-
path: /spec/kubeadmConfigSpec/files/-
value:
name: preKubeadmScript
# Note: This file has to be escaped as it is run through envsubst by clusterctl
value: |
owner: root:root
path: "/etc/pre-kubeadm-commands/20-k8s-install.sh"
permissions: "0755"
content: |
#!/bin/bash
set -o nounset
set -o pipefail
set -o errexit
DISTRO="ubuntu"
KUBE_BINARY_DIR="/usr/bin"
source /etc/lsb-release
if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
# Overrides for flatcar
DISTRO="flatcar"
KUBE_BINARY_DIR="/opt/bin"
fi
function retry {
attempt=0
Expand All @@ -36,8 +46,8 @@
LINE_SEPARATOR="*************************************************"
echo "$${LINE_SEPARATOR}"
## Variables (replaced by JSON patching)
KUBERNETES_VERSION=$${KUBERNETES_VERSION}
## Variables (replaced by envtest patching)
KUBERNETES_VERSION=${KUBERNETES_VERSION}
##
# Note: We assume if kubectl has the right version, everything else has as well
Expand All @@ -55,7 +65,7 @@
CONTAINER_EXT="tar"
echo "* testing CI version $${KUBERNETES_VERSION}"
# Check for semver
if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
export DEBIAN_FRONTEND=noninteractive
Expand All @@ -78,13 +88,16 @@
systemctl restart kubelet
else
CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
fi
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
# Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
# e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
done
systemctl restart kubelet
fi
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
- op: add
path: /spec/template/spec/files
value: []
- op: add
path: /spec/template/spec/files/-
value:
owner: root:root
path: "/etc/pre-kubeadm-commands/20-k8s-install.sh"
permissions: "0755"
content: |
#!/bin/bash
set -o nounset
set -o pipefail
set -o errexit
DISTRO="ubuntu"
KUBE_BINARY_DIR="/usr/bin"
source /etc/lsb-release
if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
# Overrides for flatcar
DISTRO="flatcar"
KUBE_BINARY_DIR="/opt/bin"
fi
function retry {
attempt=0
max_attempts=$${1}
interval=$${2}
shift; shift
until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
attempt=$((attempt+1))
set +e
eval "$*" && return || echo "failed $${attempt} times: $*"
set -e
sleep "$${interval}"
done
echo "error: reached max attempts at retry($*)"
return 1
}
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
# This test installs release packages or binaries that are a result of the CI and release builds.
# It runs '... --version' commands to verify that the binaries are correctly installed
# and finally uninstalls the packages.
# For the release packages it tests all versions in the support skew.
LINE_SEPARATOR="*************************************************"
echo "$${LINE_SEPARATOR}"
## Variables (replaced by envtest patching)
KUBERNETES_VERSION=${KUBERNETES_VERSION}
##
# Note: We assume if kubectl has the right version, everything else has as well
if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then
echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do"
exit 0
fi
if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
CI_DIR=/tmp/k8s-ci
mkdir -p "$${CI_DIR}"
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
# Let's just also download the control plane images for worker nodes. It's easier then optimizing it.
declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
CONTAINER_EXT="tar"
echo "* testing CI version $${KUBERNETES_VERSION}"
# Check for semver
if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
export DEBIAN_FRONTEND=noninteractive
# sometimes the network is not immediately available, so we have to retry the apt-get update
retry 10 5 "apt-get update"
apt-get install -y apt-transport-https ca-certificates curl gpg
mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
apt-get update
# replace . with \.
VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
apt-mark unhold "$${CI_PACKAGE}"
apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
apt-mark hold "$${CI_PACKAGE}"
done
systemctl restart kubelet
else
CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
fi
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
# Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
# e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
done
systemctl restart kubelet
fi
IMAGE_REGISTRY_PREFIX=registry.k8s.io
# Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io
if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then
IMAGE_REGISTRY_PREFIX=k8s.gcr.io
fi
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
$${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
$${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
$${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
done
fi
echo "* checking binary versions"
echo "ctr version: " "$(ctr version)"
echo "kubeadm version: " "$(kubeadm version -o=short)"
echo "kubectl version: " "$(kubectl version --client=true)"
echo "kubelet version: " "$(kubelet --version)"
echo "$${LINE_SEPARATOR}"
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ patches:
- target:
kind: ClusterClass
path: ./patch-prekubeadmscript.yaml
- target:
kind: ClusterClass
path: ./patch-k8s-install-script.yaml
- target:
kind: ClusterClass
path: ./patch-namingstrategy.yaml
Loading

0 comments on commit 169889a

Please sign in to comment.