From 513fcc55d4102e82babc3ef9001737710b5abf6c Mon Sep 17 00:00:00 2001 From: Sachin Kumar Singh Date: Wed, 22 Feb 2023 12:31:14 +0530 Subject: [PATCH] Add e2e tests for in-place upgrades (#65) --- integration/README.md | 11 ++- integration/e2e_test.go | 156 +++++++++++++++++++++++++++------------- 2 files changed, 115 insertions(+), 52 deletions(-) diff --git a/integration/README.md b/integration/README.md index f28518a..a905784 100644 --- a/integration/README.md +++ b/integration/README.md @@ -15,17 +15,24 @@ The integration/e2e tests have the following prerequisites: export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large export AWS_NODE_MACHINE_FLAVOR=t3.large export CLUSTER_NAME=test-ci-cluster - clusterctl generate cluster ${CLUSTER_NAME} --from "bootstrap/templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster.yaml + clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster.yaml export CLUSTER_MANIFEST_FILE=$PWD/cluster.yaml ``` * Additional environment variables when testing cluster upgrades: ``` - export CAPI_UPGRADE_VERSION=v1.25.0 + export CAPI_UPGRADE_VERSION=v1.26.0 export CAPI_UPGRADE_MD_NAME=${CLUSTER_NAME}-md-0 export CAPI_UPGRADE_MD_TYPE=machinedeployments.cluster.x-k8s.io export CAPI_UPGRADE_CP_NAME=${CLUSTER_NAME}-control-plane export CAPI_UPGRADE_CP_TYPE=microk8scontrolplanes.controlplane.cluster.x-k8s.io + + # Change the control plane and worker machine count to desired values for in-place upgrades tests and create a new cluster manifest. + CONTROL_PLANE_MACHINE_COUNT=1 + WORKER_MACHINE_COUNT=1 + clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster-inplace.yaml + export CLUSTER_INPLACE_MANIFEST_FILE=$PWD/cluster-inplace.yaml + ``` * `clusterctl` available in the PATH diff --git a/integration/e2e_test.go b/integration/e2e_test.go index 2ae1831..a284de8 100644 --- a/integration/e2e_test.go +++ b/integration/e2e_test.go @@ -48,22 +48,51 @@ func init() { // kubectl and clusterctl have to be avaibale in the caller's path. // kubectl should be setup so it uses the kubeconfig of the management cluster by default. func TestBasic(t *testing.T) { + cluster_manifest_file := os.Getenv("CLUSTER_MANIFEST_FILE") + if cluster_manifest_file == "" { + t.Fatalf("Environment variable CLUSTER_MANIFEST_FILE is not set. " + + "CLUSTER_MANIFEST_FILE is expected to hold the PATH to a cluster manifest.") + } + t.Logf("Cluster to setup is in %s", cluster_manifest_file) + setupCheck(t) t.Cleanup(teardownCluster) - t.Run("DeployCluster", func(t *testing.T) { deployCluster(t) }) + t.Run("DeployCluster", func(t *testing.T) { deployCluster(t, os.Getenv("CLUSTER_MANIFEST_FILE")) }) t.Run("DeployMicrobot", func(t *testing.T) { deployMicrobot(t) }) - t.Run("UpgradeCluster", func(t *testing.T) { upgradeCluster(t) }) + t.Run("UpgradeClusterRollout", func(t *testing.T) { upgradeCluster(t, "RollingUpgrade") }) + + // Important: the cluster is deleted in the Cleanup function + // which is called after all subtests are finished. + t.Logf("Deleting the cluster") } -func setupCheck(t testing.TB) { - cluster_manifest_file := os.Getenv("CLUSTER_MANIFEST_FILE") - if cluster_manifest_file == "" { - t.Fatalf("Environment variable CLUSTER_MANIFEST_FILE is not set. " + - "CLUSTER_MANIFEST_FILE is expected to hold the PATH to a cluster manifest.") +// TestInPlaceUpgrade waits for the target cluster to deploy and start a 30 pod deployment. +// The CLUSTER_INPLACE_MANIFEST_FILE environment variable should point to a manifest with the target cluster +// This cluster will be upgraded via an in-place upgrade. +func TestInPlaceUpgrade(t *testing.T) { + cluster_inplace_manifest_file := os.Getenv("CLUSTER_INPLACE_MANIFEST_FILE") + if cluster_inplace_manifest_file == "" { + t.Fatalf("Environment variable CLUSTER_INPLACE_MANIFEST_FILE is not set. " + + "CLUSTER_INPLACE_MANIFEST_FILE is expected to hold the PATH to a cluster manifest for in-place upgrades.") } - t.Logf("Cluster to setup is in %s", cluster_manifest_file) + t.Logf("Cluster for in-place upgrade test setup is in %s", cluster_inplace_manifest_file) + + setupCheck(t) + t.Cleanup(teardownCluster) + t.Run("DeployCluster", func(t *testing.T) { deployCluster(t, os.Getenv("CLUSTER_INPLACE_MANIFEST_FILE")) }) + t.Run("DeployMicrobot", func(t *testing.T) { deployMicrobot(t) }) + t.Run("UpgradeClusterInplace", func(t *testing.T) { upgradeCluster(t, "InPlaceUpgrade") }) + + // Important: the cluster is deleted in the Cleanup function + // which is called after all subtests are finished. + t.Logf("Deleting the cluster") + +} + +// setupCheck checks that the environment is ready to run the tests. +func setupCheck(t testing.TB) { _, err := exec.LookPath("kubectl") if err != nil { t.Fatalf("Please make sure kubectl is in your PATH. %s", err) @@ -79,6 +108,7 @@ func setupCheck(t testing.TB) { waitForPod(t, "capi-microk8s-control-plane-controller-manager", "capi-microk8s-control-plane-system") } +// waitForPod waits for a pod to be available. func waitForPod(t testing.TB, pod string, ns string) { attempt := 0 maxAttempts := 10 @@ -101,6 +131,7 @@ func waitForPod(t testing.TB, pod string, ns string) { } } +// teardownCluster deletes the cluster created by deployCluster. func teardownCluster() { command := []string{"kubectl", "get", "cluster", "--no-headers", "-o", "custom-columns=:metadata.name"} cmd := exec.Command(command[0], command[1:]...) @@ -116,9 +147,10 @@ func teardownCluster() { cmd.Run() } -func deployCluster(t testing.TB) { +// deployCluster deploys a cluster using the manifest in CLUSTER_MANIFEST_FILE. +func deployCluster(t testing.TB, cluster_manifest_file string) { t.Log("Setting up the cluster") - command := []string{"kubectl", "apply", "-f", os.Getenv("CLUSTER_MANIFEST_FILE")} + command := []string{"kubectl", "apply", "-f", cluster_manifest_file} cmd := exec.Command(command[0], command[1:]...) outputBytes, err := cmd.CombinedOutput() if err != nil { @@ -256,6 +288,7 @@ func deployCluster(t testing.TB) { } } +// deployMicrobot deploys a deployment of microbot. func deployMicrobot(t testing.TB) { t.Log("Deploying microbot") @@ -290,53 +323,22 @@ func deployMicrobot(t testing.TB) { } } -func upgradeCluster(t testing.TB) { - version := os.Getenv("CAPI_UPGRADE_VERSION") - if version == "" { - t.Fatalf("Environment variable CAPI_UPGRADE_VERSION is not set." + - "Please set it to the version you want to upgrade to.") - } - - control_plane_name := os.Getenv("CAPI_UPGRADE_CP_NAME") - if control_plane_name == "" { - t.Fatalf("Environment variable CAPI_UPGRADE_CP_NAME is not set." + - "Please set it to the name of the control plane you want to upgrade.") - } - - control_plane_type := os.Getenv("CAPI_UPGRADE_CP_TYPE") - if control_plane_type == "" { - t.Fatalf("Environment variable CAPI_UPGRADE_CP_TYPE is not set." + - "Please set it to the type of the control plane you want to upgrade.") - } +// upgradeCluster upgrades the cluster to a new version based on the upgrade strategy. +func upgradeCluster(t testing.TB, upgrade_strategy string) { - worker_deployment_name := os.Getenv("CAPI_UPGRADE_MD_NAME") - if worker_deployment_name == "" { - t.Fatalf("Environment variable CAPI_UPGRADE_MD_NAME is not set." + - "Please set it to the name of the machine deployment you want to upgrade.") - } - - worker_deployment_type := os.Getenv("CAPI_UPGRADE_MD_TYPE") - if worker_deployment_type == "" { - t.Fatalf("Environment variable CAPI_UPGRADE_MD_TYPE is not set." + - "Please set it to the type of the machine deployment you want to upgrade.") - } + version, control_plane_name, control_plane_type, worker_deployment_name, + worker_deployment_type := getUpgradeEnvVars(t) - t.Logf("Upgrading cluster to %s", version) - // Patch contol plane machine upgrades. - command := []string{"kubectl", "patch", "--type=merge", control_plane_type, control_plane_name, "--patch", - fmt.Sprintf(`{"spec":{"version":"%s"}}`, version)} - cmd := exec.Command(command[0], command[1:]...) - outputBytes, err := cmd.CombinedOutput() + t.Logf("Upgrading cluster to %s via %s", version, upgrade_strategy) + // Patch contol plane machine upgrades based on type of upgrade strategy. + outputBytes, err := controlPlanePatch(control_plane_name, control_plane_type, version, upgrade_strategy) if err != nil { t.Error(string(outputBytes)) t.Fatalf("Failed to merge the patch to control plane. %s", err) } // Patch worker machine upgrades. - command = []string{"kubectl", "patch", "--type=merge", worker_deployment_type, worker_deployment_name, "--patch", - fmt.Sprintf(`{"spec":{"template":{"spec":{"version":"%s"}}}}`, version)} - cmd = exec.Command(command[0], command[1:]...) - outputBytes, err = cmd.CombinedOutput() + outputBytes, err = workerPatch(worker_deployment_name, worker_deployment_type, version) if err != nil { t.Error(string(outputBytes)) t.Fatalf("Failed to merge the patch to the machine deployments. %s", err) @@ -347,7 +349,7 @@ func upgradeCluster(t testing.TB) { // Now all the machines should be upgraded to the new version. attempt := 0 maxAttempts := 60 - command = []string{"kubectl", "get", "machine", "--no-headers"} + command := []string{"kubectl", "get", "machine", "--no-headers"} for { cmd := exec.Command(command[0], command[1:]...) outputBytes, err := cmd.CombinedOutput() @@ -378,3 +380,57 @@ func upgradeCluster(t testing.TB) { } } } + +// controlPlanePatch patches the control plane machines based on the upgrade strategy and version. +func controlPlanePatch(control_plane_name, control_plane_type, version, upgrade_strategy string) ([]byte, error) { + command := []string{"kubectl", "patch", "--type=merge", control_plane_type, control_plane_name, "--patch", + fmt.Sprintf(`{"spec":{"version":"%s","upgradeStrategy":"%s"}}`, version, upgrade_strategy)} + cmd := exec.Command(command[0], command[1:]...) + + return cmd.CombinedOutput() +} + +// workerPatch patches a given worker machines with the given version. +func workerPatch(worker_deployment_name, worker_deployment_type, version string) ([]byte, error) { + command := []string{"kubectl", "patch", "--type=merge", worker_deployment_type, worker_deployment_name, "--patch", + fmt.Sprintf(`{"spec":{"template":{"spec":{"version":"%s"}}}}`, version)} + cmd := exec.Command(command[0], command[1:]...) + + return cmd.CombinedOutput() +} + +// getUpgradeEnvVars returns the environment variables needed for the upgrade test. +func getUpgradeEnvVars(t testing.TB) (version string, control_plane_name string, control_plane_type string, + worker_deployment_name string, worker_deployment_type string) { + version = os.Getenv("CAPI_UPGRADE_VERSION") + if version == "" { + t.Fatalf("Environment variable CAPI_UPGRADE_VERSION is not set." + + "Please set it to the version you want to upgrade to.") + } + + control_plane_name = os.Getenv("CAPI_UPGRADE_CP_NAME") + if control_plane_name == "" { + t.Fatalf("Environment variable CAPI_UPGRADE_CP_NAME is not set." + + "Please set it to the name of the control plane you want to upgrade.") + } + + control_plane_type = os.Getenv("CAPI_UPGRADE_CP_TYPE") + if control_plane_type == "" { + t.Fatalf("Environment variable CAPI_UPGRADE_CP_TYPE is not set." + + "Please set it to the type of the control plane you want to upgrade.") + } + + worker_deployment_name = os.Getenv("CAPI_UPGRADE_MD_NAME") + if worker_deployment_name == "" { + t.Fatalf("Environment variable CAPI_UPGRADE_MD_NAME is not set." + + "Please set it to the name of the machine deployment you want to upgrade.") + } + + worker_deployment_type = os.Getenv("CAPI_UPGRADE_MD_TYPE") + if worker_deployment_type == "" { + t.Fatalf("Environment variable CAPI_UPGRADE_MD_TYPE is not set." + + "Please set it to the type of the machine deployment you want to upgrade.") + } + + return version, control_plane_name, control_plane_type, worker_deployment_name, worker_deployment_type +}