Skip to content

Commit

Permalink
Add e2e tests for in-place upgrades (#65)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sachinkumarsingh092
sachinkumarsingh092 authored Feb 22, 2023
1 parent f730324 commit 513fcc5
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 52 deletions.
11 changes: 9 additions & 2 deletions integration/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,24 @@ The integration/e2e tests have the following prerequisites:
export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large
export AWS_NODE_MACHINE_FLAVOR=t3.large
export CLUSTER_NAME=test-ci-cluster
clusterctl generate cluster ${CLUSTER_NAME} --from "bootstrap/templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster.yaml
clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster.yaml
export CLUSTER_MANIFEST_FILE=$PWD/cluster.yaml
```

* Additional environment variables when testing cluster upgrades:
```
export CAPI_UPGRADE_VERSION=v1.25.0
export CAPI_UPGRADE_VERSION=v1.26.0
export CAPI_UPGRADE_MD_NAME=${CLUSTER_NAME}-md-0
export CAPI_UPGRADE_MD_TYPE=machinedeployments.cluster.x-k8s.io
export CAPI_UPGRADE_CP_NAME=${CLUSTER_NAME}-control-plane
export CAPI_UPGRADE_CP_TYPE=microk8scontrolplanes.controlplane.cluster.x-k8s.io
# Change the control plane and worker machine count to desired values for in-place upgrades tests and create a new cluster manifest.
CONTROL_PLANE_MACHINE_COUNT=1
WORKER_MACHINE_COUNT=1
clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster-inplace.yaml
export CLUSTER_INPLACE_MANIFEST_FILE=$PWD/cluster-inplace.yaml
```

* `clusterctl` available in the PATH
Expand Down
156 changes: 106 additions & 50 deletions integration/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,22 +48,51 @@ func init() {
// kubectl and clusterctl have to be avaibale in the caller's path.
// kubectl should be setup so it uses the kubeconfig of the management cluster by default.
func TestBasic(t *testing.T) {
cluster_manifest_file := os.Getenv("CLUSTER_MANIFEST_FILE")
if cluster_manifest_file == "" {
t.Fatalf("Environment variable CLUSTER_MANIFEST_FILE is not set. " +
"CLUSTER_MANIFEST_FILE is expected to hold the PATH to a cluster manifest.")
}
t.Logf("Cluster to setup is in %s", cluster_manifest_file)

setupCheck(t)
t.Cleanup(teardownCluster)

t.Run("DeployCluster", func(t *testing.T) { deployCluster(t) })
t.Run("DeployCluster", func(t *testing.T) { deployCluster(t, os.Getenv("CLUSTER_MANIFEST_FILE")) })
t.Run("DeployMicrobot", func(t *testing.T) { deployMicrobot(t) })
t.Run("UpgradeCluster", func(t *testing.T) { upgradeCluster(t) })
t.Run("UpgradeClusterRollout", func(t *testing.T) { upgradeCluster(t, "RollingUpgrade") })

// Important: the cluster is deleted in the Cleanup function
// which is called after all subtests are finished.
t.Logf("Deleting the cluster")
}

func setupCheck(t testing.TB) {
cluster_manifest_file := os.Getenv("CLUSTER_MANIFEST_FILE")
if cluster_manifest_file == "" {
t.Fatalf("Environment variable CLUSTER_MANIFEST_FILE is not set. " +
"CLUSTER_MANIFEST_FILE is expected to hold the PATH to a cluster manifest.")
// TestInPlaceUpgrade waits for the target cluster to deploy and start a 30 pod deployment.
// The CLUSTER_INPLACE_MANIFEST_FILE environment variable should point to a manifest with the target cluster
// This cluster will be upgraded via an in-place upgrade.
func TestInPlaceUpgrade(t *testing.T) {
cluster_inplace_manifest_file := os.Getenv("CLUSTER_INPLACE_MANIFEST_FILE")
if cluster_inplace_manifest_file == "" {
t.Fatalf("Environment variable CLUSTER_INPLACE_MANIFEST_FILE is not set. " +
"CLUSTER_INPLACE_MANIFEST_FILE is expected to hold the PATH to a cluster manifest for in-place upgrades.")
}
t.Logf("Cluster to setup is in %s", cluster_manifest_file)
t.Logf("Cluster for in-place upgrade test setup is in %s", cluster_inplace_manifest_file)

setupCheck(t)
t.Cleanup(teardownCluster)

t.Run("DeployCluster", func(t *testing.T) { deployCluster(t, os.Getenv("CLUSTER_INPLACE_MANIFEST_FILE")) })
t.Run("DeployMicrobot", func(t *testing.T) { deployMicrobot(t) })
t.Run("UpgradeClusterInplace", func(t *testing.T) { upgradeCluster(t, "InPlaceUpgrade") })

// Important: the cluster is deleted in the Cleanup function
// which is called after all subtests are finished.
t.Logf("Deleting the cluster")

}

// setupCheck checks that the environment is ready to run the tests.
func setupCheck(t testing.TB) {
_, err := exec.LookPath("kubectl")
if err != nil {
t.Fatalf("Please make sure kubectl is in your PATH. %s", err)
Expand All @@ -79,6 +108,7 @@ func setupCheck(t testing.TB) {
waitForPod(t, "capi-microk8s-control-plane-controller-manager", "capi-microk8s-control-plane-system")
}

// waitForPod waits for a pod to be available.
func waitForPod(t testing.TB, pod string, ns string) {
attempt := 0
maxAttempts := 10
Expand All @@ -101,6 +131,7 @@ func waitForPod(t testing.TB, pod string, ns string) {
}
}

// teardownCluster deletes the cluster created by deployCluster.
func teardownCluster() {
command := []string{"kubectl", "get", "cluster", "--no-headers", "-o", "custom-columns=:metadata.name"}
cmd := exec.Command(command[0], command[1:]...)
Expand All @@ -116,9 +147,10 @@ func teardownCluster() {
cmd.Run()
}

func deployCluster(t testing.TB) {
// deployCluster deploys a cluster using the manifest in CLUSTER_MANIFEST_FILE.
func deployCluster(t testing.TB, cluster_manifest_file string) {
t.Log("Setting up the cluster")
command := []string{"kubectl", "apply", "-f", os.Getenv("CLUSTER_MANIFEST_FILE")}
command := []string{"kubectl", "apply", "-f", cluster_manifest_file}
cmd := exec.Command(command[0], command[1:]...)
outputBytes, err := cmd.CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -256,6 +288,7 @@ func deployCluster(t testing.TB) {
}
}

// deployMicrobot deploys a deployment of microbot.
func deployMicrobot(t testing.TB) {
t.Log("Deploying microbot")

Expand Down Expand Up @@ -290,53 +323,22 @@ func deployMicrobot(t testing.TB) {
}
}

func upgradeCluster(t testing.TB) {
version := os.Getenv("CAPI_UPGRADE_VERSION")
if version == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_VERSION is not set." +
"Please set it to the version you want to upgrade to.")
}

control_plane_name := os.Getenv("CAPI_UPGRADE_CP_NAME")
if control_plane_name == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_CP_NAME is not set." +
"Please set it to the name of the control plane you want to upgrade.")
}

control_plane_type := os.Getenv("CAPI_UPGRADE_CP_TYPE")
if control_plane_type == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_CP_TYPE is not set." +
"Please set it to the type of the control plane you want to upgrade.")
}
// upgradeCluster upgrades the cluster to a new version based on the upgrade strategy.
func upgradeCluster(t testing.TB, upgrade_strategy string) {

worker_deployment_name := os.Getenv("CAPI_UPGRADE_MD_NAME")
if worker_deployment_name == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_MD_NAME is not set." +
"Please set it to the name of the machine deployment you want to upgrade.")
}

worker_deployment_type := os.Getenv("CAPI_UPGRADE_MD_TYPE")
if worker_deployment_type == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_MD_TYPE is not set." +
"Please set it to the type of the machine deployment you want to upgrade.")
}
version, control_plane_name, control_plane_type, worker_deployment_name,
worker_deployment_type := getUpgradeEnvVars(t)

t.Logf("Upgrading cluster to %s", version)
// Patch contol plane machine upgrades.
command := []string{"kubectl", "patch", "--type=merge", control_plane_type, control_plane_name, "--patch",
fmt.Sprintf(`{"spec":{"version":"%s"}}`, version)}
cmd := exec.Command(command[0], command[1:]...)
outputBytes, err := cmd.CombinedOutput()
t.Logf("Upgrading cluster to %s via %s", version, upgrade_strategy)
// Patch contol plane machine upgrades based on type of upgrade strategy.
outputBytes, err := controlPlanePatch(control_plane_name, control_plane_type, version, upgrade_strategy)
if err != nil {
t.Error(string(outputBytes))
t.Fatalf("Failed to merge the patch to control plane. %s", err)
}

// Patch worker machine upgrades.
command = []string{"kubectl", "patch", "--type=merge", worker_deployment_type, worker_deployment_name, "--patch",
fmt.Sprintf(`{"spec":{"template":{"spec":{"version":"%s"}}}}`, version)}
cmd = exec.Command(command[0], command[1:]...)
outputBytes, err = cmd.CombinedOutput()
outputBytes, err = workerPatch(worker_deployment_name, worker_deployment_type, version)
if err != nil {
t.Error(string(outputBytes))
t.Fatalf("Failed to merge the patch to the machine deployments. %s", err)
Expand All @@ -347,7 +349,7 @@ func upgradeCluster(t testing.TB) {
// Now all the machines should be upgraded to the new version.
attempt := 0
maxAttempts := 60
command = []string{"kubectl", "get", "machine", "--no-headers"}
command := []string{"kubectl", "get", "machine", "--no-headers"}
for {
cmd := exec.Command(command[0], command[1:]...)
outputBytes, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -378,3 +380,57 @@ func upgradeCluster(t testing.TB) {
}
}
}

// controlPlanePatch patches the control plane machines based on the upgrade strategy and version.
func controlPlanePatch(control_plane_name, control_plane_type, version, upgrade_strategy string) ([]byte, error) {
command := []string{"kubectl", "patch", "--type=merge", control_plane_type, control_plane_name, "--patch",
fmt.Sprintf(`{"spec":{"version":"%s","upgradeStrategy":"%s"}}`, version, upgrade_strategy)}
cmd := exec.Command(command[0], command[1:]...)

return cmd.CombinedOutput()
}

// workerPatch patches a given worker machines with the given version.
func workerPatch(worker_deployment_name, worker_deployment_type, version string) ([]byte, error) {
command := []string{"kubectl", "patch", "--type=merge", worker_deployment_type, worker_deployment_name, "--patch",
fmt.Sprintf(`{"spec":{"template":{"spec":{"version":"%s"}}}}`, version)}
cmd := exec.Command(command[0], command[1:]...)

return cmd.CombinedOutput()
}

// getUpgradeEnvVars returns the environment variables needed for the upgrade test.
func getUpgradeEnvVars(t testing.TB) (version string, control_plane_name string, control_plane_type string,
worker_deployment_name string, worker_deployment_type string) {
version = os.Getenv("CAPI_UPGRADE_VERSION")
if version == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_VERSION is not set." +
"Please set it to the version you want to upgrade to.")
}

control_plane_name = os.Getenv("CAPI_UPGRADE_CP_NAME")
if control_plane_name == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_CP_NAME is not set." +
"Please set it to the name of the control plane you want to upgrade.")
}

control_plane_type = os.Getenv("CAPI_UPGRADE_CP_TYPE")
if control_plane_type == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_CP_TYPE is not set." +
"Please set it to the type of the control plane you want to upgrade.")
}

worker_deployment_name = os.Getenv("CAPI_UPGRADE_MD_NAME")
if worker_deployment_name == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_MD_NAME is not set." +
"Please set it to the name of the machine deployment you want to upgrade.")
}

worker_deployment_type = os.Getenv("CAPI_UPGRADE_MD_TYPE")
if worker_deployment_type == "" {
t.Fatalf("Environment variable CAPI_UPGRADE_MD_TYPE is not set." +
"Please set it to the type of the machine deployment you want to upgrade.")
}

return version, control_plane_name, control_plane_type, worker_deployment_name, worker_deployment_type
}

0 comments on commit 513fcc5

Please sign in to comment.