diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 9d1c9c550..122965238 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -41,7 +41,13 @@ E2E_CHART ?= ${REPO_ROOT}/charts/vsphere-cpi E2E_DATA_DIR := ${REPO_ROOT}/test/e2e/data # E2E_DATA_CAPV_VER defines the providers from which version of CAPV to use -E2E_DATA_CAPV_VER ?= release-1.0 +E2E_DATA_CAPV_VER ?= release-1.9 + +# VERSION_DEV is the version of the image used for development +VERSION_DEV ?= $(VERSION) + +# IMAGE_DEV is the image used for development +IMAGE_DEV ?= gcr.io/cloud-provider-vsphere/cpi/pr/manager all: run @@ -52,7 +58,7 @@ $(TMP_CAPV_DIR): git clone -b $(E2E_DATA_CAPV_VER) https://github.com/kubernetes-sigs/cluster-api-provider-vsphere.git $(TMP_CAPV_DIR) $(E2E_DATA_DIR): $(TMP_CAPV_DIR) - $(MAKE) -C $(TMP_CAPV_DIR) e2e-templates && cp -r $(TMP_CAPV_DIR)/test/e2e/data $(E2E_DATA_DIR) && \ + $(MAKE) -C $(TMP_CAPV_DIR) generate-e2e-templates && cp -r $(TMP_CAPV_DIR)/test/e2e/data $(E2E_DATA_DIR) && \ cp $(TMP_CAPV_DIR)/metadata.yaml $(E2E_DATA_DIR) push-ccm-image: @@ -70,3 +76,10 @@ dev: $(TOOLING_BINARIES) $(E2E_DATA_DIR) push-ccm-image clean: rm -rf $(E2E_DATA_DIR) $(TMP_CAPV_DIR) + +# run-e2e-with-dev-image is a convenience target to run the E2E tests with the development image already pushed to any +# registry (e.g. dockerhub). No need push permission of gcr.io/cloud-provider-vsphere/cpi/pr/manager +run-e2e-with-dev-image: $(TOOLING_BINARIES) $(E2E_DATA_DIR) + $(GINKGO) -v . -- --e2e.config="$(E2E_CONF_FILE_DEV)" --e2e.artifacts-folder="$(E2E_ARTIFACTS)" \ + --e2e.chart-folder="$(E2E_CHART)" --e2e.skip-resource-cleanup=true \ + --e2e.version="$(VERSION_DEV)" --e2e.image="$(IMAGE_DEV)" diff --git a/test/e2e/config/vsphere-ci.yaml b/test/e2e/config/vsphere-ci.yaml index f1f8e8fa9..7859e765e 100644 --- a/test/e2e/config/vsphere-ci.yaml +++ b/test/e2e/config/vsphere-ci.yaml @@ -6,165 +6,106 @@ # - vsphere providers: - - name: cluster-api - type: CoreProvider - versions: - - name: v0.3.25 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. - value: "http://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/core-components.yaml" - type: "url" - contract: v1alpha3 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/core-components.yaml" - type: "url" - contract: v1alpha4 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v1.0.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.1/core-components.yaml" - type: "url" - contract: v1beta1 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: kubeadm - type: BootstrapProvider - versions: - - name: v0.3.25 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/bootstrap-components.yaml" - type: "url" - contract: v1alpha3 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/bootstrap-components.yaml" - type: "url" - contract: v1alpha4 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v1.0.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.1/bootstrap-components.yaml" - type: "url" - contract: v1beta1 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" +- name: cluster-api + type: CoreProvider + versions: + - name: v1.6.1 + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/core-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/shared/main/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" - - name: kubeadm - type: ControlPlaneProvider - versions: - - name: v0.3.25 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/control-plane-components.yaml" - type: "url" - contract: v1alpha3 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/control-plane-components.yaml" - type: "url" - contract: v1alpha4 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" - - name: v1.0.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.1/control-plane-components.yaml" - type: "url" - contract: v1beta1 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/shared/metadata.yaml" +- name: kubeadm + type: BootstrapProvider + versions: + - name: v1.6.1 + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/bootstrap-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/shared/main/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" - - name: vsphere - type: InfrastructureProvider - versions: - - name: v0.7.12 - value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.7.12/infrastructure-components.yaml - type: url - contract: v1alpha3 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - # TODO: v1a3 cluster-template includes WORKLOAD_CONTROL_PLANE_ENDPOINT_IP - - sourcePath: "../data/metadata.yaml" - - sourcePath: "../data/infrastructure-vsphere/capi-upgrades/v1alpha3/cluster-template.yaml" - - name: v0.8.2 - value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.8.2/infrastructure-components.yaml - type: url - contract: v1alpha4 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - - sourcePath: "../data/metadata.yaml" - - sourcePath: "../data/infrastructure-vsphere/capi-upgrades/v1alpha4/cluster-template.yaml" - - name: v1.0.1 - value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v1.0.1/infrastructure-components.yaml - type: url - contract: v1beta1 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - # Add a cluster template - - sourcePath: "../data/infrastructure-vsphere/cluster-template.yaml" - - sourcePath: "../data/infrastructure-vsphere/cluster-template-remote-management.yaml" - - sourcePath: "../data/metadata.yaml" +- name: kubeadm + type: ControlPlaneProvider + versions: + - name: v1.6.1 + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/control-plane-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/shared/main/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + +- name: vsphere + type: InfrastructureProvider + versions: + - name: v1.9.0 + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v1.9.0/infrastructure-components.yaml" + type: "url" + contract: v1beta1 + files: + # Add a cluster template + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template.yaml" + - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml" variables: - KUBERNETES_VERSION: "v1.23.5" + KUBERNETES_VERSION: "v1.29.0" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.28.0" + KUBERNETES_VERSION_UPGRADE_TO: "v1.29.0" + CPI_IMAGE_K8S_VERSION: "v1.29.0" CNI: "./data/cni/calico/calico.yaml" EXP_CLUSTER_RESOURCE_SET: "true" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" CONTROL_PLANE_MACHINE_COUNT: 1 WORKER_MACHINE_COUNT: 1 IP_FAMILY: "IPv4" CLUSTER_CLASS_NAME: "quick-start" # Following CAPV variables should be set before testing VSPHERE_TLS_THUMBPRINT: "18:EC:35:60:54:68:92:F6:F8:92:3E:4D:11:A1:0D:13:9C:E9:3E:B6" + VSPHERE_COMPUTE_CLUSTER: "Cluster-1" VSPHERE_DATACENTER: "SDDC-Datacenter" VSPHERE_FOLDER: "cloud-provider-vsphere" VSPHERE_RESOURCE_POOL: "cloud-provider-vsphere" VSPHERE_DATASTORE: "WorkloadDatastore" VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" VSPHERE_NETWORK: "sddc-cgw-network-6" - VSPHERE_TEMPLATE: "ubuntu-2004-kube-v1.23.5" - VSPHERE_INSECURE: "true" + VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" + FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" VSPHERE_INSECURE_CSI: "true" - INIT_WITH_BINARY_V1ALPHA3: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/clusterctl-{OS}-{ARCH}" - INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/clusterctl-{OS}-{ARCH}" - INIT_WITH_KUBERNETES_VERSION: "v1.23.5" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" NODE_DRAIN_TIMEOUT: "60s" CLUSTER_TOPOLOGY: "true" + # These IDs correspond to Tesla T4s, they are the decimal representation of the hex values. + DEVICE_ID: 7864 + VENDOR_ID: 4318 + # CAPV feature flags + EXP_NODE_ANTI_AFFINITY: "true" + CAPI_DIAGNOSTICS_ADDRESS: ":8080" + CAPI_INSECURE_DIAGNOSTICS: "true" + intervals: default/wait-controllers: ["5m", "10s"] default/wait-cluster: ["5m", "10s"] - default/wait-control-plane: ["20m", "10s"] - default/wait-worker-nodes: ["20m", "10s"] + default/wait-control-plane: ["10m", "10s"] + default/wait-worker-nodes: ["10m", "10s"] default/wait-delete-cluster: ["5m", "10s"] default/wait-machine-upgrade: ["15m", "1m"] + default/wait-machine-remediation: ["15m", "10s"] + mhc-remediation/mhc-remediation: ["30m", "10s"] + node-drain/wait-deployment-available: ["3m", "10s"] + node-drain/wait-machine-deleted: ["2m", "10s"] + anti-affinity/wait-vm-redistribution: ["5m", "10s"] diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b746505b3..865584e86 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -63,6 +63,9 @@ var ( // If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically. clusterctlConfig string + // image is the cloud-controller-manager image to be tested, for example, gcr.io/cloud-provider-vsphere/cpi/pr/manager + image string + // version is the cloud-controller-manager version to be tested, for example, v1.22.3-76-g6f4fa01 version string @@ -84,7 +87,6 @@ var ( // helm install configurations namespace = "kube-system" release = "vsphere-cpi-e2e" - image = "gcr.io/cloud-provider-vsphere/cpi/pr/manager" // helm install expectation daemonsetName = "vsphere-cpi" @@ -95,6 +97,7 @@ func init() { flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.StringVar(&clusterctlConfig, "e2e.clusterctl-config", "", "file which tests will use as a clusterctl config. If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically.") flag.StringVar(&chartFolder, "e2e.chart-folder", "", "folder where the helm chart for e2e should be stored") + flag.StringVar(&image, "e2e.image", "gcr.io/cloud-provider-vsphere/cpi/pr/manager", "the cloud-controller-manager image to be tested, for example, gcr.io/cloud-provider-vsphere/cpi/pr/manager") flag.StringVar(&version, "e2e.version", "dev", "the cloud-controller-manager version to be tested, for example, v1.22.3-76-g6f4fa01") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") @@ -192,7 +195,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, ClusterName: workloadName, Namespace: workloadKubeconfigNamespace, - KubernetesVersion: e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"), + KubernetesVersion: e2eConfig.GetVariable("KUBERNETES_VERSION"), ControlPlaneMachineCount: e2eConfig.GetInt64PtrVariable("CONTROL_PLANE_MACHINE_COUNT"), WorkerMachineCount: e2eConfig.GetInt64PtrVariable("WORKER_MACHINE_COUNT"), Flavor: clusterctl.DefaultFlavor, diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 97ed9dab7..300d6c719 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -11,8 +11,8 @@ require ( k8s.io/apimachinery v0.29.0 k8s.io/client-go v0.29.0 k8s.io/klog/v2 v2.110.1 - sigs.k8s.io/cluster-api v1.6.0-rc.0.0.20240112131553-005971d34c42 - sigs.k8s.io/cluster-api/test v1.6.0-rc.0.0.20240112131553-005971d34c42 + sigs.k8s.io/cluster-api v1.6.1 + sigs.k8s.io/cluster-api/test v1.6.1 ) require ( @@ -35,7 +35,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/containerd v1.7.6 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -145,6 +145,8 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect + go.etcd.io/etcd/client/v3 v3.5.11 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect go.opentelemetry.io/otel/trace v1.20.0 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index f7d71e88b..f12a2f135 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -68,8 +68,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= @@ -704,10 +704,10 @@ oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= -sigs.k8s.io/cluster-api v1.6.0-rc.0.0.20240112131553-005971d34c42 h1:ZX4sixXAvPgWpxhyN5Wq9RyczFf3B+JD98IbUFBZIXg= -sigs.k8s.io/cluster-api v1.6.0-rc.0.0.20240112131553-005971d34c42/go.mod h1:If8MYjC8hD3muVu8j7aEePEEJMGtVkt6E3txAVxMQwc= -sigs.k8s.io/cluster-api/test v1.6.0-rc.0.0.20240112131553-005971d34c42 h1:MFZpo5uzyIKj1lFYRf0fJhHZVLFgrP0UMVHpeavXjN0= -sigs.k8s.io/cluster-api/test v1.6.0-rc.0.0.20240112131553-005971d34c42/go.mod h1:HIlOrT+EdgRq61rqZN7/eGY4Yg09vfnolw/TVxsNYrM= +sigs.k8s.io/cluster-api v1.6.1 h1:I34p/fwgRlEhs+o9cUhKXDwNNfPS3no0yJsd2bJyQVc= +sigs.k8s.io/cluster-api v1.6.1/go.mod h1:DaxwruDvSaEYq5q6FREDaGzX6UsAVUCA99Sp8vfMHyQ= +sigs.k8s.io/cluster-api/test v1.6.1 h1:9TffRPOuYNUyfHqdeWQtFhdK0oY+NAbvjlzbqK7chTw= +sigs.k8s.io/cluster-api/test v1.6.1/go.mod h1:+zOSrnG/2wI2XtWOkaVpVJ1BXumT/73zqRXZBYrclPQ= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=