Cleanup CloudStack network resources pre-test

cmd/eks-a-tool/cmd/cloudstackrmvms.go

@@ -25,7 +25,7 @@ var cloudstackRmVmsCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		err = cleanup.CleanUpCloudstackTestResources(cmd.Context(), clusterName, viper.GetBool(dryRunFlag))
+		err = cleanup.CloudStackRmVms(cmd.Context(), clusterName, viper.GetBool(dryRunFlag))
 		if err != nil {
 			log.Fatalf("Error removing vms: %v", err)
 		}

cmd/integration_test/build/buildspecs/cloudstack-test-eks-a-cli.yml

@@ -81,7 +81,8 @@ phases:
         fi
       - >
         ./bin/test e2e cleanup cloudstack
-        -n ${CLUSTER_NAME_PREFIX}
+        -c ${CLUSTER_NAME_PREFIX}
+        -n "${T_CLOUDSTACK_NETWORK},${T_CLOUDSTACK_NETWORK_2},${T_CLOUDSTACK_NETWORK_3}"
         -v 4
   build:
     commands:

cmd/integration_test/cmd/cleanupcloudstack.go

@@ -37,12 +37,16 @@ func preRunCleanUpCloudstackSetup(cmd *cobra.Command, args []string) {
 	})
 }
 
-var requiredCloudstackCleanUpFlags = []string{clusterNameFlagName}
+var requiredCloudstackCleanUpFlags = []string{
+	clusterNameFlagName,
+	networkNameFlagName,
+}
 
 func init() {
 	cleanUpInstancesCmd.AddCommand(cleanUpCloudstackCmd)
-	cleanUpCloudstackCmd.Flags().StringP(clusterNameFlagName, "n", "", "Cluster name for associated vms")
-
+	cleanUpCloudstackCmd.Flags().StringP(clusterNameFlagName, "c", "", "Cluster name for associated vms")
+	cleanUpCloudstackCmd.Flags().StringP(networkNameFlagName, "n", "", "Comma seperated network names to be cleaned")
+	cleanUpCloudstackCmd.Flags().Bool(dryRunFlagName, false, "Dry run")
 	for _, flag := range requiredCloudstackCleanUpFlags {
 		if err := cleanUpCloudstackCmd.MarkFlagRequired(flag); err != nil {
 			log.Fatalf("Error marking flag %s as required: %v", flag, err)
@@ -52,7 +56,9 @@ func init() {
 
 func cleanUpCloudstackTestResources(ctx context.Context) error {
 	clusterName := viper.GetString(clusterNameFlagName)
-	err := cleanup.CleanUpCloudstackTestResources(ctx, clusterName, false)
+	networkNames := viper.GetString(networkNameFlagName)
+	dryRun := viper.GetBool(dryRunFlagName)
+	err := cleanup.CleanUpCloudstackTestResources(ctx, clusterName, networkNames, dryRun)
 	if err != nil {
 		return fmt.Errorf("running cleanup for cloudstack vms: %v", err)
 	}

cmd/integration_test/cmd/cleanupvsphere.go

@@ -15,6 +15,8 @@ import (
 
 const (
 	clusterNameFlagName = "cluster-name"
+	networkNameFlagName = "network-name"
+	dryRunFlagName      = "dry-run"
 )
 
 var cleanUpVsphereCmd = &cobra.Command{

internal/test/cleanup/cleanup.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -22,8 +23,11 @@ import (
 )
 
 const (
-	cleanupRetries = 5
-	retryBackoff   = 10 * time.Second
+	cleanupRetries              = 5
+	retryBackoff                = 10 * time.Second
+	cloudStackNetworkType       = "Shared"
+	cloudStackNetworkReadyState = "Setup"
+	cloudStackAdminProfileName  = "global"
 )
 
 func CleanUpAwsTestResources(storageBucket string, maxAge string, tag string) error {
@@ -95,25 +99,16 @@ func VsphereRmVms(ctx context.Context, clusterName string, opts ...executables.G
 	return govc.CleanupVms(ctx, clusterName, false)
 }
 
-func CleanUpCloudstackTestResources(ctx context.Context, clusterName string, dryRun bool) error {
-	executableBuilder, close, err := executables.InitInDockerExecutablesBuilder(ctx, executables.DefaultEksaImage())
-	if err != nil {
-		return fmt.Errorf("unable to initialize executables: %v", err)
-	}
-	defer close.CheckErr(ctx)
-	tmpWriter, err := filewriter.NewWriter("rmvms")
-	if err != nil {
-		return fmt.Errorf("creating filewriter for directory rmvms: %v", err)
-	}
-	execConfig, err := decoder.ParseCloudStackCredsFromEnv()
-	if err != nil {
-		return fmt.Errorf("parsing cloudstack credentials from environment: %v", err)
-	}
-	cmk, err := executableBuilder.BuildCmkExecutable(tmpWriter, execConfig)
+func CloudStackRmVms(ctx context.Context, clusterName string, dryRun bool) error {
+	cmk, close, execConfig, err := initCmk(ctx)
 	if err != nil {
-		return fmt.Errorf("building cmk executable: %v", err)
+		return err
 	}
-	defer cmk.Close(ctx)
+	defer func() {
+		cmk.Close(ctx)
+		close.CheckErr(ctx)
+	}()
+
 	cleanupRetrier := retrier.NewWithMaxRetries(cleanupRetries, retryBackoff)
 
 	errorsMap := map[string]error{}
@@ -124,13 +119,89 @@ func CleanUpCloudstackTestResources(ctx context.Context, clusterName string, dry
 			errorsMap[profile.Name] = err
 		}
 	}
-
 	if len(errorsMap) > 0 {
 		return fmt.Errorf("cleaning up VMs: %+v", errorsMap)
 	}
 	return nil
 }
 
+func cleanUpCloudStackNetwork(ctx context.Context, networkNames string, dryRun bool) error {
+	cmk, close, execConfig, err := initCmk(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cmk.Close(ctx)
+		close.CheckErr(ctx)
+	}()
+	cleanupRetrier := retrier.NewWithMaxRetries(cleanupRetries, retryBackoff)
+
+	for _, profile := range execConfig.Profiles {
+		if profile.Name == cloudStackAdminProfileName {
+			networks := strings.Split(networkNames, ",")
+			for _, network := range networks {
+				logger.Info("Cleaning network", "network name", network)
+				// Network cleanup. Delete router
+				if err := cmk.DeleteVirtualRouter(ctx, profile.Name, network, dryRun); err != nil {
+					return err
+				}
+
+				// Restart network and wait till network status is "Setup"
+				logger.V(4).Info("Restarting network", "network name", network)
+				if err := cmk.RestartNetwork(ctx, profile.Name, network, cloudStackNetworkType, dryRun); err != nil {
+					return err
+				}
+				if err := cleanupRetrier.Retry(func() error {
+					cloudstackNetwork, err := cmk.GetNetwork(ctx, profile.Name, network, cloudStackNetworkType)
+					if err != nil {
+						return err
+					}
+					if cloudstackNetwork.State != cloudStackNetworkReadyState {
+						return fmt.Errorf("network has not finished restarting")
+					}
+					return nil
+				}); err != nil {
+					return err
+				}
+				logger.Info("Finished cleaning network", "network name", network)
+			}
+		}
+	}
+	return nil
+}
+
+func CleanUpCloudstackTestResources(ctx context.Context, clusterName, networkNames string, dryRun bool) error {
+	err := CloudStackRmVms(ctx, clusterName, dryRun)
+	if err != nil {
+		return err
+	}
+	err = cleanUpCloudStackNetwork(ctx, networkNames, dryRun)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func initCmk(ctx context.Context) (*executables.Cmk, executables.Closer, *decoder.CloudStackExecConfig, error) {
+	executableBuilder, close, err := executables.InitInDockerExecutablesBuilder(ctx, executables.DefaultEksaImage())
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("unable to initialize executables: %v", err)
+	}
+	tmpWriter, err := filewriter.NewWriter("rmvms")
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("creating filewriter for directory rmvms: %v", err)
+	}
+	execConfig, err := decoder.ParseCloudStackCredsFromEnv()
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("parsing cloudstack credentials from environment: %v", err)
+	}
+	cmk, err := executableBuilder.BuildCmkExecutable(tmpWriter, execConfig)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("building cmk executable: %v", err)
+	}
+	return cmk, close, execConfig, nil
+}
+
 // NutanixTestResourcesCleanup cleans up any leftover VMs in Nutanix after a test run.
 func NutanixTestResourcesCleanup(ctx context.Context, clusterName, endpoint, port string, insecure, ignoreErrors bool) error {
 	creds := nutanix.GetCredsFromEnv()

pkg/executables/cmk.go

@@ -395,6 +395,93 @@ func (c *Cmk) GetManagementApiEndpoint(profile string) (string, error) {
 	return "", fmt.Errorf("profile %s does not exist", profile)
 }
 
+// DeleteVirtualRouter finds the virtual router for the provided network name and deletes the router only if found.
+func (c *Cmk) DeleteVirtualRouter(ctx context.Context, profile, networkName string, dryRun bool) error {
+	command := newCmkCommand("list routers")
+	applyCmkArgs(&command, withCloudStackKeyword(networkName), appendArgs("listall=true"))
+	result, err := c.exec(ctx, profile, command...)
+	if err != nil {
+		return fmt.Errorf("listing virtual routers with networkName %s: %s: %v", networkName, result.String(), err)
+	}
+	response := struct {
+		VirtualRouters []cmkResourceIdentifier `json:"router"`
+	}{}
+	if err = json.Unmarshal(result.Bytes(), &response); err != nil {
+		return fmt.Errorf("parsing response into json: %v", err)
+	}
+	if len(response.VirtualRouters) == 0 {
+		logger.Info("virtual routers not found", "network name", networkName)
+		return nil
+	}
+	for _, router := range response.VirtualRouters {
+		if dryRun {
+			logger.Info("Found ", "virtual router name", router.Name)
+			continue
+		}
+		destroyCommand := newCmkCommand("destroy router")
+		applyCmkArgs(&destroyCommand, withCloudStackId(router.Id))
+		destroyResult, err := c.exec(ctx, profile, destroyCommand...)
+		if err != nil {
+			return fmt.Errorf("destroying virtual router with name %s and id %s: %s: %v", router.Name, router.Id, destroyResult.String(), err)
+		}
+		logger.Info("Deleted ", "router name:", router.Name, "router id", router.Id)
+	}
+	return nil
+}
+
+// RestartNetwork finds the network on CloudStack with provided name, if found will restart the network.
+func (c *Cmk) RestartNetwork(ctx context.Context, profile, networkName, networkType string, dryRun bool) error {
+	network, err := c.GetNetwork(ctx, profile, networkName, networkType)
+	if err != nil {
+		return err
+	}
+	if network == nil {
+		logger.Info("No network found to restart", "network name", networkName)
+		return nil
+	}
+
+	if dryRun {
+		logger.Info("Found ", "network name", network.Name, "network id", network.Id)
+		return nil
+	}
+	restartCommand := newCmkCommand("restart network")
+	applyCmkArgs(&restartCommand, withCloudStackId(network.Id))
+	result, err := c.exec(ctx, profile, restartCommand...)
+	if err != nil {
+		return fmt.Errorf("restarting network %s: %s: %v", network.Name, result.String(), err)
+	}
+	logger.Info("Restarted", "network name", network.Name, "network id", network.Id)
+
+	return nil
+}
+
+// GetNetwork searches CloudStack for network with provided name and type and returns the network object only when
+// one network was found with the provided name. Returns error if multiple networks with same name are found.
+// Returns nil if no networks are found.
+func (c *Cmk) GetNetwork(ctx context.Context, profile, networkName, networkType string) (*cmkNetwork, error) {
+	command := newCmkCommand("list networks")
+	applyCmkArgs(&command, withCloudStackName(networkName), withCloudStackNetworkType(networkType), appendArgs("listall=true"))
+	result, err := c.exec(ctx, profile, command...)
+	if err != nil {
+		return nil, fmt.Errorf("listing networks with name %s: %s: %v", networkName, result.String(), err)
+	}
+	response := struct {
+		Networks []cmkNetwork `json:"network"`
+	}{}
+	if err = json.Unmarshal(result.Bytes(), &response); err != nil {
+		return nil, fmt.Errorf("parsing response into json: %v", err)
+	}
+	if len(response.Networks) == 0 {
+		logger.Info("networks not found", "networkName", networkName)
+		return nil, nil
+	}
+	if len(response.Networks) > 1 {
+		logger.Info("More then one network found", "network name", networkName)
+		return nil, fmt.Errorf("multiple networks with same name found")
+	}
+	return &response.Networks[0], nil
+}
+
 func (c *Cmk) CleanupVms(ctx context.Context, profile string, clusterName string, dryRun bool) error {
 	command := newCmkCommand("list virtualmachines")
 	applyCmkArgs(&command, withCloudStackKeyword(clusterName), appendArgs("listall=true"))
@@ -495,6 +582,13 @@ type cmkResourceIdentifier struct {
 	Name string `json:"name"`
 }
 
+type cmkNetwork struct {
+	State string `json:"state"`
+	Type  string `json:"type"`
+
+	cmkResourceIdentifier
+}
+
 type cmkDiskOffering struct {
 	Id         string `json:"id"`
 	Name       string `json:"name"`

pkg/executables/cmk_cmd_builder.go

@@ -46,3 +46,7 @@ func withCloudStackName(name string) cmkCommandArgs {
 func withCloudStackKeyword(keyword string) cmkCommandArgs {
 	return appendArgs(fmt.Sprintf("keyword=\"%s\"", keyword))
 }
+
+func withCloudStackNetworkType(networkType string) cmkCommandArgs {
+	return appendArgs(fmt.Sprintf("type=\"%s\"", networkType))
+}