From f4f0203135e7c120e9fe4f5e1d48fa99711fb3bf Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Apr 2024 22:46:08 -0700 Subject: [PATCH] Nutanix ensure VMs cleaned up before and after tests --- cmd/eks-a-tool/cmd/nutanixrmvms.go | 2 +- .../buildspecs/nutanix-test-eks-a-cli.yml | 18 +++++ .../build/buildspecs/quick-test-eks-a-cli.yml | 16 ++++ cmd/integration_test/cmd/cleanupnutanix.go | 73 +++++++++++++++++++ internal/test/cleanup/cleanup.go | 4 +- test/framework/nutanix.go | 2 +- 6 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 cmd/integration_test/cmd/cleanupnutanix.go diff --git a/cmd/eks-a-tool/cmd/nutanixrmvms.go b/cmd/eks-a-tool/cmd/nutanixrmvms.go index 87ddf517739c6..c68b4c8c97d7e 100644 --- a/cmd/eks-a-tool/cmd/nutanixrmvms.go +++ b/cmd/eks-a-tool/cmd/nutanixrmvms.go @@ -31,7 +31,7 @@ var nutanixRmVmsCmd = &cobra.Command{ if viper.IsSet(insecureFlag) { insecure = true } - err = cleanup.NutanixTestResourcesCleanup(cmd.Context(), clusterName, viper.GetString(endpointFlag), viper.GetString(portFlag), insecure, viper.GetBool(ignoreErrorsFlag)) + err = cleanup.NutanixTestResources(cmd.Context(), clusterName, viper.GetString(endpointFlag), viper.GetString(portFlag), insecure, viper.GetBool(ignoreErrorsFlag)) if err != nil { log.Fatalf("Error removing vms: %v", err) } diff --git a/cmd/integration_test/build/buildspecs/nutanix-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/nutanix-test-eks-a-cli.yml index 10f3491c6b767..e264fdab0c132 100644 --- a/cmd/integration_test/build/buildspecs/nutanix-test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/nutanix-test-eks-a-cli.yml @@ -59,6 +59,14 @@ phases: if ! [[ ${CODEBUILD_INITIATOR} =~ "codepipeline" ]]; then make build-eks-a-for-e2e build-integration-test-binary e2e-tests-binary E2E_TAGS="e2e nutanix" E2E_OUTPUT_FILE=bin/nutanix/e2e.test fi + - > + ./bin/test e2e cleanup nutanix + -n ${CLUSTER_NAME_PREFIX} + -e ${T_NUTANIX_ENDPOINT} + -p ${T_NUTANIX_PORT} + --insecure + --ignoreErrors + -v 4 build: commands: - export JOB_ID=$CODEBUILD_BUILD_ID @@ -85,6 +93,16 @@ phases: --test-report-folder=reports --branch-name=${BRANCH_NAME} --baremetal-branch=${BAREMETAL_BRANCH} + post_build: + commands: + - > + ./bin/test e2e cleanup nutanix + -n ${CLUSTER_NAME_PREFIX} + -e ${T_NUTANIX_ENDPOINT} + -p ${T_NUTANIX_PORT} + --insecure + --ignoreErrors + -v 4 reports: e2e-reports: files: diff --git a/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml index 8afa3db618a18..2c519ed25e662 100644 --- a/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml @@ -179,6 +179,14 @@ phases: -n ${CLUSTER_NAME_PREFIX} --delete-duplicate-networks -v 6 + - > + ./bin/test e2e cleanup nutanix + -n ${CLUSTER_NAME_PREFIX} + -e ${T_NUTANIX_ENDPOINT} + -p ${T_NUTANIX_PORT} + --insecure + --ignoreErrors + -v 4 build: commands: - export JOB_ID=$CODEBUILD_BUILD_ID @@ -219,6 +227,14 @@ phases: ./bin/test e2e cleanup cloudstack -n ${CLUSTER_NAME_PREFIX} -v 4 + - > + ./bin/test e2e cleanup nutanix + -n ${CLUSTER_NAME_PREFIX} + -e ${T_NUTANIX_ENDPOINT} + -p ${T_NUTANIX_PORT} + --insecure + --ignoreErrors + -v 4 reports: e2e-reports: files: diff --git a/cmd/integration_test/cmd/cleanupnutanix.go b/cmd/integration_test/cmd/cleanupnutanix.go new file mode 100644 index 0000000000000..883f655ef84bb --- /dev/null +++ b/cmd/integration_test/cmd/cleanupnutanix.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "context" + "fmt" + "log" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/aws/eks-anywhere/internal/test/cleanup" + "github.com/aws/eks-anywhere/pkg/logger" +) + +const ( + endpointFlag = "endpoint" + portFlag = "port" + insecureFlag = "insecure" + ignoreErrorsFlag = "ignoreErrors" +) + +var requiredNutanixCleanUpFlags = []string{clusterNameFlagName, endpointFlag} + +var cleanUpNutanixCmd = &cobra.Command{ + Use: "nutanix", + Short: "Clean up e2e vms on Nutanix Prism", + Long: "Clean up vms created for e2e testing on Nutanix Prism", + SilenceUsage: true, + PreRun: preRunCleanUpNutanixSetup, + RunE: func(cmd *cobra.Command, args []string) error { + err := cleanUpNutanixTestResources(cmd.Context()) + if err != nil { + logger.Fatal(err, "Failed to cleanup e2e vms on Nutanix Prism") + } + return nil + }, +} + +func preRunCleanUpNutanixSetup(cmd *cobra.Command, args []string) { + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + err := viper.BindPFlag(flag.Name, flag) + if err != nil { + log.Fatalf("Error initializing flags: %v", err) + } + }) +} + +func init() { + cleanUpInstancesCmd.AddCommand(cleanUpNutanixCmd) + + cleanUpNutanixCmd.Flags().StringP(clusterNameFlagName, "n", "", "Cluster name for associated vms") + cleanUpNutanixCmd.Flags().StringP(endpointFlag, "e", "", "Nutanix Prism endpoint") + cleanUpNutanixCmd.Flags().StringP(portFlag, "p", "9440", "Nutanix Prism port") + cleanUpNutanixCmd.Flags().BoolP(insecureFlag, "k", false, "skip TLS when contacting Prism APIs") + cleanUpNutanixCmd.Flags().Bool(ignoreErrorsFlag, true, "ignore APIs errors when deleting VMs") + + for _, flag := range requiredNutanixCleanUpFlags { + if err := cleanUpNutanixCmd.MarkFlagRequired(flag); err != nil { + log.Fatalf("Error marking flag %s as required: %v", flag, err) + } + } +} + +func cleanUpNutanixTestResources(ctx context.Context) error { + clusterName := viper.GetString(clusterNameFlagName) + err := cleanup.NutanixTestResources(ctx, clusterName, viper.GetString(endpointFlag), viper.GetString(portFlag), viper.IsSet(insecureFlag), viper.IsSet(ignoreErrorsFlag)) + if err != nil { + return fmt.Errorf("running cleanup for vsphere vcenter vms: %v", err) + } + + return nil +} diff --git a/internal/test/cleanup/cleanup.go b/internal/test/cleanup/cleanup.go index 0e3271dcfb537..2362819b9f915 100644 --- a/internal/test/cleanup/cleanup.go +++ b/internal/test/cleanup/cleanup.go @@ -154,8 +154,8 @@ func cleanupCloudstackDuplicateNetworks(ctx context.Context, cmk *executables.Cm return nil } -// NutanixTestResourcesCleanup cleans up any leftover VMs in Nutanix after a test run. -func NutanixTestResourcesCleanup(ctx context.Context, clusterName, endpoint, port string, insecure, ignoreErrors bool) error { +// NutanixTestResources cleans up any leftover VMs in Nutanix after a test run. +func NutanixTestResources(ctx context.Context, clusterName, endpoint, port string, insecure, ignoreErrors bool) error { creds := nutanix.GetCredsFromEnv() nutanixCreds := prismgoclient.Credentials{ URL: fmt.Sprintf("%s:%s", endpoint, port), diff --git a/test/framework/nutanix.go b/test/framework/nutanix.go index 314a9c47304bc..0bed30e527b3e 100644 --- a/test/framework/nutanix.go +++ b/test/framework/nutanix.go @@ -160,7 +160,7 @@ func (n *Nutanix) UpdateKubeConfig(content *[]byte, clusterName string) error { // CleanupVMs satisfies the test framework Provider. func (n *Nutanix) CleanupVMs(clustername string) error { - return cleanup.NutanixTestResourcesCleanup(context.Background(), clustername, os.Getenv(nutanixEndpoint), os.Getenv(nutanixPort), true, true) + return cleanup.NutanixTestResources(context.Background(), clustername, os.Getenv(nutanixEndpoint), os.Getenv(nutanixPort), true, true) } // ClusterConfigUpdates satisfies the test framework Provider.