initial aws virtuals (#175)

* use subdomain when present

* add air file

* cloudflare token adjustments, aws virtuals

* path adjustment

* accommodate AWS GitHub structure

* add write secrets to aws provider create

* add new bootstrap secets to aws

* updates for aws gitlab

* Update extensions/aws/secrets.go

.air.toml

@@ -0,0 +1,36 @@
+root = "."
+testdata_dir = "testdata"
+tmp_dir = "tmp"
+
+[build]
+  bin = "./tmp/main"
+  cmd = "go build -o ./tmp/main ."
+  delay = 1000
+  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
+  exclude_file = []
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = true
+  full_bin = ""
+  include_dir = []
+  include_ext = ["go", "tpl", "tmpl", "html"]
+  kill_delay = "0s"
+  log = "build-errors.log"
+  send_interrupt = false
+  stop_on_error = true
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  time = true
+
+[misc]
+  clean_on_exit = false
+
+[screen]
+  clear_on_rebuild = false

extensions/aws/env.go

@@ -98,6 +98,8 @@ func GetVaultTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster
 	envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset)
 	envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret
 	envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL
+	envs["TF_VAR_cloudflare_origin_ca_api_key"] = cl.CloudflareAuth.OriginCaIssuerKey
+	envs["TF_VAR_cloudflare_api_key"] = cl.CloudflareAuth.APIToken
 	envs["TF_VAR_kbot_ssh_private_key"] = cl.GitAuth.PrivateKey
 	envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey
 

extensions/aws/secrets.go

@@ -26,15 +26,14 @@ func BootstrapAWSMgmtCluster(
 	destinationGitopsRepoURL string,
 	awsClient *aws.AWSConfiguration,
 ) error {
-
 
 	err := providerConfig.BootstrapMgmtCluster(
 		clientset,
 		cl.GitProvider,
 		cl.GitAuth.User,
 		destinationGitopsRepoURL,
 		cl.GitProtocol,
-		cl.CloudflareAuth.Token,
+		cl.CloudflareAuth.APIToken,
 		"",
 		cl.DnsProvider,
 		cl.CloudProvider,
@@ -45,9 +44,61 @@ func BootstrapAWSMgmtCluster(
 		log.Fatal().Msgf("error in central function to create secrets: %s", err)
 		return err
 	}
+	var externalDnsToken string
+	switch cl.DnsProvider {
+	case "civo":
+		externalDnsToken = cl.CivoAuth.Token
+	case "vultr":
+		externalDnsToken = cl.VultrAuth.Token
+	case "digitalocean":
+		externalDnsToken = cl.DigitaloceanAuth.Token
+	case "aws":
+		externalDnsToken = "implement with cluster management"
+	case "google":
+		externalDnsToken = "implement with cluster management"
+	case "cloudflare":
+		externalDnsToken = cl.CloudflareAuth.APIToken
+	}
 
 	// Create secrets
-	createSecrets := []*v1.Secret{}
+	createSecrets := []*v1.Secret{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "argo"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "atlantis"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "chartmuseum"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "external-dns-secrets", Namespace: "external-dns"},
+			Data: map[string][]byte{
+				"token": []byte(externalDnsToken),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "kubefirst"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "vault"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+	}
 	for _, secret := range createSecrets {
 		_, err := clientset.CoreV1().Secrets(secret.ObjectMeta.Namespace).Get(context.TODO(), secret.ObjectMeta.Name, metav1.GetOptions{})
 		if err == nil {

internal/controller/argocd.go

@@ -226,6 +226,10 @@ func (clctrl *ClusterController) DeployRegistryApplication() error {
 			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 		} else if clctrl.CloudProvider == "civo" && clctrl.GitProvider == "gitlab" {
 			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+		} else if clctrl.CloudProvider == "aws" && clctrl.GitProvider == "github" {
+			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+		} else if clctrl.CloudProvider == "aws" && clctrl.GitProvider == "gitlab" {
+			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 		} else {
 			registryPath = fmt.Sprintf("registry/%s", clctrl.ClusterName)
 		}

internal/controller/kms.go

@@ -45,6 +45,10 @@ func (clctrl *ClusterController) DetokenizeKMSKeyID() error {
 				registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 			} else if clctrl.CloudProvider == "civo" && clctrl.GitProvider == "gitlab" {
 				registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+			} else if clctrl.CloudProvider == "aws" && clctrl.GitProvider == "github" {
+				registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+			} else if clctrl.CloudProvider == "aws" && clctrl.GitProvider == "gitlab" {
+				registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 			} else {
 				registryPath = fmt.Sprintf("registry/%s", clctrl.ClusterName)
 			}

internal/services/services.go

@@ -71,6 +71,8 @@ func CreateService(cl *pkgtypes.Cluster, serviceName string, appDef *types.Gitop
 		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
 	} else if cl.CloudProvider == "civo" && cl.GitProvider == "gitlab" {
 		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
+	} else if cl.CloudProvider == "aws" && cl.GitProvider == "github" {
+		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
 	} else {
 		registryPath = fmt.Sprintf("registry/%s", cl.ClusterName)
 	}
@@ -262,6 +264,8 @@ func DeleteService(cl *pkgtypes.Cluster, serviceName string) error {
 		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
 	} else if cl.CloudProvider == "civo" && cl.GitProvider == "gitlab" {
 		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
+	} else if cl.CloudProvider == "aws" && cl.GitProvider == "github" {
+		registryPath = fmt.Sprintf("registry/clusters/%s", cl.ClusterName)
 	} else {
 		registryPath = fmt.Sprintf("registry/%s", cl.ClusterName)
 	}

pkg/providerConfigs/adjustDriver.go

@@ -51,6 +51,81 @@ func AdjustGitopsRepo(
 
 		},
 	}
+
+	AWS_GITHUB := "aws-github"
+
+	if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == AWS_GITHUB {
+		driverContent := fmt.Sprintf("%s/%s-%s/", gitopsRepoDir, cloudProvider, gitProvider)
+		err := cp.Copy(driverContent, gitopsRepoDir, opt)
+		if err != nil {
+			log.Info().Msgf("Error populating gitops repository with driver content: %s. error: %s", fmt.Sprintf("%s-%s", cloudProvider, gitProvider), err.Error())
+			return err
+		}
+		os.RemoveAll(driverContent)
+
+		//* copy $HOME/.k1/gitops/templates/${clusterType}/* $HOME/.k1/gitops/registry/${clusterName}
+		clusterContent := fmt.Sprintf("%s/templates/%s", gitopsRepoDir, clusterType)
+
+		// Remove apex content if apex content already exists
+		if apexContentExists {
+			log.Warn().Msgf("removing nginx-apex since apexContentExists was %v", apexContentExists)
+			os.Remove(fmt.Sprintf("%s/nginx-apex.yaml", clusterContent))
+			os.RemoveAll(fmt.Sprintf("%s/nginx-apex", clusterContent))
+		} else {
+			log.Warn().Msgf("will create nginx-apex since apexContentExists was %v", apexContentExists)
+		}
+
+		if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == AWS_GITHUB {
+			err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/clusters/%s", gitopsRepoDir, clusterName), opt)
+		} else {
+			err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/%s", gitopsRepoDir, clusterName), opt)
+		}
+		if err != nil {
+			log.Info().Msgf("Error populating cluster content with %s. error: %s", clusterContent, err.Error())
+			return err
+		}
+		os.RemoveAll(fmt.Sprintf("%s/templates/mgmt", gitopsRepoDir))
+
+		return nil
+	}
+
+	AWS_GITLAB := "aws-gitlab"
+
+	if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == AWS_GITLAB {
+		driverContent := fmt.Sprintf("%s/%s-%s/", gitopsRepoDir, cloudProvider, gitProvider)
+		err := cp.Copy(driverContent, gitopsRepoDir, opt)
+		if err != nil {
+			log.Info().Msgf("Error populating gitops repository with driver content: %s. error: %s", fmt.Sprintf("%s-%s", cloudProvider, gitProvider), err.Error())
+			return err
+		}
+		os.RemoveAll(driverContent)
+
+		//* copy $HOME/.k1/gitops/templates/${clusterType}/* $HOME/.k1/gitops/registry/${clusterName}
+		clusterContent := fmt.Sprintf("%s/templates/%s", gitopsRepoDir, clusterType)
+
+		// Remove apex content if apex content already exists
+		if apexContentExists {
+			log.Warn().Msgf("removing nginx-apex since apexContentExists was %v", apexContentExists)
+			os.Remove(fmt.Sprintf("%s/nginx-apex.yaml", clusterContent))
+			os.RemoveAll(fmt.Sprintf("%s/nginx-apex", clusterContent))
+		} else {
+			log.Warn().Msgf("will create nginx-apex since apexContentExists was %v", apexContentExists)
+		}
+
+		if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == AWS_GITLAB {
+			err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/clusters/%s", gitopsRepoDir, clusterName), opt)
+		} else {
+			err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/%s", gitopsRepoDir, clusterName), opt)
+		}
+		if err != nil {
+			log.Info().Msgf("Error populating cluster content with %s. error: %s", clusterContent, err.Error())
+			return err
+		}
+		os.RemoveAll(fmt.Sprintf("%s/templates/mgmt", gitopsRepoDir))
+
+		return nil
+	}
+
 	CIVO_GITHUB := "civo-github" //! i know i know i know.
 
 	if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == CIVO_GITHUB {
@@ -192,6 +267,106 @@ func AdjustMetaphorRepo(
 		},
 	}
 
+	AWS_GITHUB := "aws-github"
+
+	if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) != AWS_GITHUB {
+		os.RemoveAll(metaphorDir + "/.argo")
+		os.RemoveAll(metaphorDir + "/.github")
+	}
+
+	//todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from
+	// gitops and then allows gitops to commit during its sequence of ops
+	if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) == AWS_GITHUB {
+		//* metaphor app source
+		metaphorContent := fmt.Sprintf("%s/metaphor", gitopsRepoDir)
+		err = cp.Copy(metaphorContent, metaphorDir, opt)
+		if err != nil {
+			log.Info().Msgf("Error populating metaphor content with %s. error: %s", metaphorContent, err.Error())
+			return err
+		}
+
+		// Remove metaphor content from gitops repository directory
+		os.RemoveAll(fmt.Sprintf("%s/metaphor", gitopsRepoDir))
+
+		err = gitClient.Commit(metaphorRepo, "init commit pre ref change")
+		if err != nil {
+			return err
+		}
+
+		metaphorRepo, err = gitClient.SetRefToMainBranch(metaphorRepo)
+		if err != nil {
+			return err
+		}
+
+		// remove old git ref
+		err = metaphorRepo.Storer.RemoveReference(plumbing.NewBranchReferenceName("master"))
+		if err != nil {
+			return fmt.Errorf("error removing previous git ref: %s", err)
+		}
+
+		// create remote
+		_, err = metaphorRepo.CreateRemote(&config.RemoteConfig{
+			Name: "origin",
+			URLs: []string{destinationMetaphorRepoURL},
+		})
+		if err != nil {
+			return fmt.Errorf("error creating remote for metaphor repository: %s", err)
+		}
+
+		return nil
+
+	}
+
+	AWS_GITLAB := "aws-gitlab"
+
+	if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) != AWS_GITLAB {
+		os.RemoveAll(metaphorDir + "/.argo")
+		os.RemoveAll(metaphorDir + "/.github")
+	}
+
+	//todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from
+	// gitops and then allows gitops to commit during its sequence of ops
+	if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) == AWS_GITLAB {
+		//* metaphor app source
+		metaphorContent := fmt.Sprintf("%s/metaphor", gitopsRepoDir)
+		err = cp.Copy(metaphorContent, metaphorDir, opt)
+		if err != nil {
+			log.Info().Msgf("Error populating metaphor content with %s. error: %s", metaphorContent, err.Error())
+			return err
+		}
+
+		// Remove metaphor content from gitops repository directory
+		os.RemoveAll(fmt.Sprintf("%s/metaphor", gitopsRepoDir))
+
+		err = gitClient.Commit(metaphorRepo, "init commit pre ref change")
+		if err != nil {
+			return err
+		}
+
+		metaphorRepo, err = gitClient.SetRefToMainBranch(metaphorRepo)
+		if err != nil {
+			return err
+		}
+
+		// remove old git ref
+		err = metaphorRepo.Storer.RemoveReference(plumbing.NewBranchReferenceName("master"))
+		if err != nil {
+			return fmt.Errorf("error removing previous git ref: %s", err)
+		}
+
+		// create remote
+		_, err = metaphorRepo.CreateRemote(&config.RemoteConfig{
+			Name: "origin",
+			URLs: []string{destinationMetaphorRepoURL},
+		})
+		if err != nil {
+			return fmt.Errorf("error creating remote for metaphor repository: %s", err)
+		}
+
+		return nil
+
+	}
+
 	CIVO_GITHUB := "civo-github"
 
 	if strings.ToLower(fmt.Sprintf("civo-%s", gitProvider)) != CIVO_GITHUB {

providers/aws/create.go

@@ -205,6 +205,12 @@ func CreateAWSCluster(definition *pkgtypes.ClusterDefinition) error {
 		return err
 	}
 
+	err = ctrl.WriteVaultSecrets()
+	if err != nil {
+		ctrl.HandleError(err.Error())
+		return err
+	}
+
 	err = ctrl.RunUsersTerraform()
 	if err != nil {
 		ctrl.HandleError(err.Error())
@@ -253,7 +259,6 @@ func CreateAWSCluster(definition *pkgtypes.ClusterDefinition) error {
 
 	}
 
-
 	//* export and import cluster
 	err = ctrl.ExportClusterRecord()
 	if err != nil {