diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..d054c4e1 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,86 @@ +name: E2E Tests + +on: + pull_request: + +permissions: + contents: read + +jobs: + build-e2e-images: + name: Build & Run E2E Images + runs-on: [self-hosted, linux, X64, jammy, large] + steps: + - name: Check out repo + uses: actions/checkout@v4 + - name: Install requirements + run: | + sudo apt update + sudo snap install go --classic --channel=1.22/stable + sudo apt install make + sudo apt install docker-buildx + sudo snap install kubectl --classic --channel=1.30/stable + - name: Build provider images + run: sudo make docker-build-e2e + - name: Build k8s-snap image + run: | + cd templates/docker + sudo docker build . -t k8s-snap:dev + - name: Save provider image + run: | + sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + sudo chmod 775 provider-images.tar + - name: Save k8s-snap image + run: | + sudo docker save -o k8s-snap-image.tar k8s-snap:dev + sudo chmod 775 k8s-snap-image.tar + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-images + path: | + provider-images.tar + k8s-snap-image.tar + + run-e2e-tests: + name: Run E2E Tests + runs-on: [self-hosted, linux, X64, jammy, large] + needs: build-e2e-images + strategy: + matrix: + ginkgo_focus: + - "KCP remediation" + - "MachineDeployment remediation" + - "Workload cluster creation" + - "Workload cluster scaling" + - "Workload cluster upgrade" + steps: + - name: Check out repo + uses: actions/checkout@v4 + - name: Install requirements + run: | + sudo apt update + sudo snap install go --classic --channel=1.22/stable + sudo apt install make + sudo apt install docker-buildx + sudo snap install kubectl --classic --channel=1.30/stable + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: e2e-images + path: . + - name: Load provider image + run: sudo docker load -i provider-images.tar + - name: Load k8s-snap image + run: sudo docker load -i k8s-snap-image.tar + - name: Create docker network + run: | + sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true + - name: Increase inotify watches + run: | + # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files + sudo sysctl fs.inotify.max_user_watches=1048576 + sudo sysctl fs.inotify.max_user_instances=8192 + - name: Run e2e tests + run: | + sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e diff --git a/.golangci.yml b/.golangci.yml index b5b83e39..317c18df 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -90,6 +90,7 @@ linters-settings: - github.com/google/uuid - github.com/pkg/errors + - sigs.k8s.io/kind/pkg/errors gci: sections: - standard diff --git a/Makefile b/Makefile index b6b9d017..ad2b7688 100644 --- a/Makefile +++ b/Makefile @@ -242,6 +242,9 @@ docker-build-bootstrap-%: DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}-$* docker-build-bootstrap: manager-bootstrap docker-build-bootstrap-amd64 docker-build-bootstrap-arm64 +docker-build-bootstrap-e2e: manager-bootstrap + DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG} + # Push the bootstrap multiarch image .PHONY: docker-push-bootstrap docker-push-bootstrap-%: docker-build-bootstrap-% @@ -266,8 +269,8 @@ test-controlplane: envtest generate-controlplane generate-controlplane-conversio docker-build-e2e: ## Run docker-build-* targets for all the images with settings to be used for the e2e tests # please ensure the generated image name matches image names used in the E2E_CONF_FILE # and it also match the image tags in bootstrap/config/default and controlplane/config/default - $(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap - $(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane + $(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap-e2e + $(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane-e2e .PHONY: test-e2e test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests @@ -326,6 +329,9 @@ docker-build-controlplane-%: DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}-$* docker-build-controlplane: manager-controlplane docker-build-controlplane-amd64 docker-build-controlplane-arm64 +docker-build-controlplane-e2e: manager-controlplane + DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=${ARCH} --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG} + # Push the controlplane multiarch image .PHONY: docker-push-controlplane docker-push-controlplane-%: docker-build-controlplane-% diff --git a/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml b/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml index 6d3fc62c..f1afdde5 100644 --- a/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml +++ b/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml @@ -1,4 +1,4 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfig metadata: name: ck8sconfig-sample diff --git a/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml b/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml index f336faf1..eb3029ad 100644 --- a/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml +++ b/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml @@ -1,4 +1,4 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate metadata: name: ck8sconfigtemplate-sample diff --git a/controlplane/config/default/kustomization.yaml b/controlplane/config/default/kustomization.yaml index f09a39aa..89fccd20 100644 --- a/controlplane/config/default/kustomization.yaml +++ b/controlplane/config/default/kustomization.yaml @@ -28,7 +28,7 @@ patchesStrategicMerge: # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml +#- manager_auth_proxy_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml b/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml index b11f9088..5a0f24d2 100644 --- a/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml +++ b/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml @@ -1,4 +1,4 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ck8scontrolplane-sample diff --git a/docs/main.md b/docs/main.md index e7ef1853..93ca10b6 100644 --- a/docs/main.md +++ b/docs/main.md @@ -14,7 +14,7 @@ The default behaviour is to `snap install k8s` using the matching track (e.g. in You can override this behaviour by changing the default installation script by setting the following fields on the config template: ```yaml -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane @@ -47,7 +47,7 @@ For airgap deployments, or environment you can specify `airGapped: true` to prev Any extra yaml files placed in `/capi/manifests` will be applied once on the cluster after bootstrapping. Files are applied in alphabetical order, so you can use this in case of dependencies. Example: ```yaml -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane diff --git a/go.mod b/go.mod index cad35252..6ce797ce 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,7 @@ require ( go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.19.0 golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect diff --git a/go.sum b/go.sum index 1d4b1078..a03034f8 100644 --- a/go.sum +++ b/go.sum @@ -337,6 +337,8 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqR golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/test/e2e/README.md b/test/e2e/README.md index a3e3dce8..849be79c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1,27 +1,29 @@ # e2e test + The e2e test use the [Cluster API test framework](https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework?tab=doc) and use the [CAPD](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/infrastructure/docker) as the infrastructure provider. Please make sure you have [Docker](https://docs.docker.com/install/) and [kind](https://kind.sigs.k8s.io/) installed. -You could refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information. +Refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information. ## Run the e2e test + The e2e image will be built with tag `dev`. You should build the image first before running the test. + ```shell make docker-build-e2e # should be run everytime you change the controller code make test-e2e # run all e2e tests ``` + ### Run a specific e2e test + To run a specific e2e test, such as `[PR-Blocking]`, use the `GINKGO_FOCUS` environment variable as shown below: + ```shell make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e # only run e2e test with `[PR-Blocking]` in its spec name ``` -### Run the e2e test with tilt -It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run: -```shell -# running e2e for the cluster pointed by the current context -make USE_EXISTING_CLUSTER=true test-e2e -``` + ## Develop an e2e test -You could refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests. + +Refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests. A guide for developing a ck8s e2e test: @@ -31,4 +33,5 @@ A guide for developing a ck8s e2e test: * If reusing a [cluster-api test spec](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e), note that they assume the use of `KubeadmControlPlane`. For customization, copy code into `test/e2e/helpers.go`. ## Troubleshooting + * [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files) diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index e5b290d2..a058b65c 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -25,6 +25,11 @@ import ( ) var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { + BeforeEach(func() { + // TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap. + Skip("Skipping the upgrade tests as snap does not work on CAPD.") + }) + Context("Upgrading a cluster with 1 control plane", func() { ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { return ClusterUpgradeSpecInput{ diff --git a/test/e2e/config/ck8s-docker.yaml b/test/e2e/config/ck8s-docker.yaml index 011e6bdb..a622c8c1 100644 --- a/test/e2e/config/ck8s-docker.yaml +++ b/test/e2e/config/ck8s-docker.yaml @@ -64,6 +64,9 @@ providers: # is modified - name: v0.1.99 # next; use manifest from source files value: "../../../bootstrap/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev" files: - sourcePath: "../../../metadata.yaml" targetName: "metadata.yaml" @@ -72,6 +75,9 @@ providers: versions: - name: v0.1.99 # next; use manifest from source files value: "../../../controlplane/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev" files: - sourcePath: "../../../metadata.yaml" targetName: "metadata.yaml" diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml index a8677d02..8c8cb43b 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml @@ -13,7 +13,7 @@ spec: - 10.46.0.0/16 serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -27,19 +27,24 @@ metadata: name: ${CLUSTER_NAME} spec: {} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane namespace: ${NAMESPACE} spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane spec: + airGapped: true + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" files: - path: /wait-signal.sh content: | @@ -62,19 +67,14 @@ spec: echo "signal $signal" if [ "$signal" == "pass" ]; then - curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test - exit 0 + curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test + exit 0 fi done permissions: "0777" - preK3sCommands: + owner: root:root + preRunCommands: - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" - serverConfig: - tlsSan: - - localhost - - 127.0.0.1 - - 0.0.0.0 - - host.docker.internal --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate @@ -84,7 +84,7 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + customImage: k8s-snap:dev --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -111,7 +111,7 @@ spec: clusterName: ${CLUSTER_NAME} bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate name: ${CLUSTER_NAME}-md-0 infrastructureRef: @@ -127,9 +127,9 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + customImage: k8s-snap:dev --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml index 3bf354eb..7b2bfe64 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml @@ -16,7 +16,7 @@ spec: - 10.46.0.0/16 serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -30,22 +30,24 @@ metadata: name: ${CLUSTER_NAME} spec: {} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane namespace: ${NAMESPACE} spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + airGapped: true + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} - spec: - serverConfig: - tlsSan: - - 0.0.0.0 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate @@ -55,7 +57,7 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + customImage: k8s-snap:dev --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -83,7 +85,7 @@ spec: clusterName: ${CLUSTER_NAME} bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate name: ${CLUSTER_NAME}-md-0 infrastructureRef: @@ -99,9 +101,9 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + customImage: k8s-snap:dev --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 @@ -109,6 +111,7 @@ metadata: spec: template: spec: + airGapped: true --- # MachineHealthCheck object with # - a selector that targets all the machines with label e2e.remediation.label="" diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index 062164cb..b33ac49d 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -10,13 +10,13 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.45.0.0/16 + - 10.1.0.0/16 services: cidrBlocks: - - 10.46.0.0/16 + - 10.152.0.0/16 serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -30,22 +30,24 @@ metadata: name: ${CLUSTER_NAME} spec: {} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane namespace: ${NAMESPACE} spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + airGapped: true + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} - spec: - serverConfig: - tlsSan: - - 0.0.0.0 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate @@ -55,7 +57,8 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + # TODO: make this customable + customImage: k8s-snap:dev --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -82,7 +85,7 @@ spec: clusterName: ${CLUSTER_NAME} bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate name: ${CLUSTER_NAME}-md-0 infrastructureRef: @@ -98,9 +101,9 @@ metadata: spec: template: spec: - customImage: kindest/node:${KIND_IMAGE_VERSION} + customImage: k8s-snap:dev --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 @@ -108,3 +111,4 @@ metadata: spec: template: spec: + airGapped: true diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 24e8dec1..5c0cc56a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -27,10 +27,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" + "golang.org/x/mod/semver" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/noderefutil" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -587,6 +590,35 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont }, input.WaitForMachinesToBeUpgraded...) } +type WaitForNodesReadyInput struct { + Lister framework.Lister + KubernetesVersion string + Count int + WaitForNodesReady []interface{} +} + +// WaitForNodesReady waits until there are exactly the given count nodes and they have the correct Kubernetes minor version +// and are ready. +func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) { + Eventually(func() (bool, error) { + nodeList := &corev1.NodeList{} + if err := input.Lister.List(ctx, nodeList); err != nil { + return false, err + } + nodeReadyCount := 0 + for _, node := range nodeList.Items { + if !(semver.MajorMinor(node.Status.NodeInfo.KubeletVersion) == semver.MajorMinor(input.KubernetesVersion)) { + return false, nil + } + if !noderefutil.IsNodeReady(&node) { + return false, nil + } + nodeReadyCount++ + } + return input.Count == nodeReadyCount, nil + }, input.WaitForNodesReady...).Should(BeTrue()) +} + // byClusterOptions returns a set of ListOptions that allows to identify all the objects belonging to a Cluster. func byClusterOptions(name, namespace string) []client.ListOption { return []client.ListOption{ diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go index 9b2c9d4f..4f707ba2 100644 --- a/test/e2e/md_remediation_test.go +++ b/test/e2e/md_remediation_test.go @@ -108,7 +108,8 @@ var _ = Describe("When testing MachineDeployment remediation", func() { By("Waiting until nodes are ready") workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name) workloadClient := workloadProxy.GetClient() - framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{ + + WaitForNodesReady(ctx, WaitForNodesReadyInput{ Lister: workloadClient, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), Count: int(result.ExpectedTotalNodes()),