Add k3s backend (#281)

* feat: add k3s backend Signed-off-by: DrummyFloyd <[email protected]> Signed-off-by: DrummyFloyd <[email protected]> * add crossplane secrets to vault and create ns --------- Signed-off-by: DrummyFloyd <[email protected]> Signed-off-by: DrummyFloyd <[email protected]> Co-authored-by: Jared Edwards <[email protected]>
konstructio · Mar 11, 2024 · 5293e10 · 5293e10
1 parent 4cfe32b
commit 5293e10
Show file tree

Hide file tree

Showing 22 changed files with 826 additions and 68 deletions.
diff --git a/extensions/k3s/env.go b/extensions/k3s/env.go
@@ -0,0 +1,118 @@
+/*
+Copyright (C) 2021-2023, Kubefirst
+
+This program is licensed under MIT.
+See the LICENSE file for more details.
+*/
+package k3s
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/kubefirst/kubefirst-api/pkg/providerConfigs"
+	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
+	"github.com/kubefirst/runtime/pkg/k8s"
+	"github.com/kubefirst/runtime/pkg/vault"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
+)
+
+func readVaultTokenFromSecret(clientset *kubernetes.Clientset) string {
+	existingKubernetesSecret, err := k8s.ReadSecretV2(clientset, vault.VaultNamespace, vault.VaultSecretName)
+	if err != nil || existingKubernetesSecret == nil {
+		log.Printf("Error reading existing Secret data: %s", err)
+		return ""
+	}
+
+	return existingKubernetesSecret["root-token"]
+}
+
+func GetK3sTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string {
+	envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID
+	envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["AWS_SESSION_TOKEN"] = "" // allows for debugging
+	envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID
+	envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_aws_session_token"] = "" // allows for debugging
+	// envs["TF_LOG"] = "debug"
+
+	return envs
+}
+
+func GetGithubTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string {
+	envs["GITHUB_TOKEN"] = cl.GitAuth.Token
+	envs["GITHUB_OWNER"] = cl.GitAuth.Owner
+	envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret
+	envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey
+	envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID
+	envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID
+	envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["AWS_SESSION_TOKEN"] = ""        // allows for debugging
+	envs["TF_VAR_aws_session_token"] = "" // allows for debugging
+
+	return envs
+}
+
+func GetGitlabTerraformEnvs(envs map[string]string, gid int, cl *pkgtypes.Cluster) map[string]string {
+	envs["GITLAB_TOKEN"] = cl.GitAuth.Token
+	envs["GITLAB_OWNER"] = cl.GitAuth.Owner
+	envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret
+	envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL
+	envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey
+	envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID
+	envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID
+	envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_owner_group_id"] = strconv.Itoa(gid)
+	envs["TF_VAR_gitlab_owner"] = cl.GitAuth.Owner
+	envs["AWS_SESSION_TOKEN"] = ""        // allows for debugging
+	envs["TF_VAR_aws_session_token"] = "" // allows for debugging
+
+	return envs
+}
+
+func GetUsersTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, envs map[string]string) map[string]string {
+	envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset)
+	envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL
+	envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token
+	envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner
+	envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID
+	envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID
+	envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["AWS_SESSION_TOKEN"] = ""        // allows for debugging
+	envs["TF_VAR_aws_session_token"] = "" // allows for debugging
+
+	return envs
+}
+
+func GetVaultTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, envs map[string]string) map[string]string {
+	envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token
+	envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner
+	envs["TF_VAR_email_address"] = cl.AlertsEmail
+	envs["TF_VAR_vault_addr"] = providerConfigs.VaultPortForwardURL
+	envs["TF_VAR_vault_token"] = readVaultTokenFromSecret(clientset)
+	envs[fmt.Sprintf("TF_VAR_%s_token", cl.GitProvider)] = cl.GitAuth.Token
+	envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL
+	envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset)
+	envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret
+	envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL
+	envs["TF_VAR_kbot_ssh_private_key"] = cl.GitAuth.PrivateKey
+	envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey
+	envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID
+	envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID
+	envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey
+	envs["AWS_SESSION_TOKEN"] = ""        // allows for debugging
+	envs["TF_VAR_aws_session_token"] = "" // allows for debugging
+
+	switch cl.GitProvider {
+	case "gitlab":
+		envs["TF_VAR_owner_group_id"] = fmt.Sprint(cl.GitlabOwnerGroupID)
+	}
+
+	return envs
+}
diff --git a/extensions/k3s/secrets.go b/extensions/k3s/secrets.go
@@ -0,0 +1,117 @@
+/*
+Copyright (C) 2021-2023, Kubefirst
+
+This program is licensed under MIT.
+See the LICENSE file for more details.
+*/
+package k3s
+
+import (
+	"context"
+	"strings"
+
+	providerConfig "github.com/kubefirst/kubefirst-api/pkg/providerConfigs"
+	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
+	"github.com/rs/zerolog/log"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func BootstrapK3sMgmtCluster(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, destinationGitopsRepoURL string) error {
+	err := providerConfig.BootstrapMgmtCluster(
+		clientset,
+		cl.GitProvider,
+		cl.GitAuth.User,
+		destinationGitopsRepoURL,
+		cl.GitProtocol,
+		cl.CloudflareAuth.Token,
+		"",
+		cl.DnsProvider,
+		cl.CloudProvider,
+		cl.GitAuth.Token,
+		cl.GitAuth.PrivateKey,
+	)
+	if err != nil {
+		log.Fatal().Msgf("error in central function to create secrets: %s", err)
+		return err
+	}
+
+	var externalDnsToken string
+	switch cl.DnsProvider {
+	case "civo":
+		externalDnsToken = cl.CivoAuth.Token
+	case "vultr":
+		externalDnsToken = cl.VultrAuth.Token
+	case "digitalocean":
+		externalDnsToken = cl.DigitaloceanAuth.Token
+	case "aws":
+		externalDnsToken = "implement with cluster management"
+	case "google":
+		externalDnsToken = "implement with cluster management"
+	case "cloudflare":
+		externalDnsToken = cl.CloudflareAuth.APIToken
+	}
+
+	// Create secrets
+	createSecrets := []*v1.Secret{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "argo"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "atlantis"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "chartmuseum"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "external-dns-secrets", Namespace: "external-dns"},
+			Data: map[string][]byte{
+				"token": []byte(externalDnsToken),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "kubefirst"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "vault"},
+			Data: map[string][]byte{
+				"origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "crossplane-secrets", Namespace: "crossplane-system"},
+			Data: map[string][]byte{
+				"username": []byte(cl.GitAuth.User),
+				"password": []byte(cl.GitAuth.Token),
+			},
+		},
+	}
+	for _, secret := range createSecrets {
+		_, err := clientset.CoreV1().Secrets(secret.ObjectMeta.Namespace).Get(context.TODO(), secret.ObjectMeta.Name, metav1.GetOptions{})
+		if err == nil {
+			log.Info().Msgf("kubernetes secret %s/%s already created - skipping", secret.Namespace, secret.Name)
+		} else if strings.Contains(err.Error(), "not found") {
+			_, err = clientset.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
+			if err != nil {
+				log.Fatal().Msgf("error creating kubernetes secret %s/%s: %s", secret.Namespace, secret.Name, err)
+			}
+			log.Info().Msgf("created kubernetes secret: %s/%s", secret.Namespace, secret.Name)
+		}
+	}
+
+	return nil
+}
diff --git a/internal/controller/argocd.go b/internal/controller/argocd.go
@@ -34,7 +34,7 @@ func (clctrl *ClusterController) InstallArgoCD() error {
 		switch clctrl.CloudProvider {
 		case "aws":
 			kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName)
-		case "civo", "digitalocean", "vultr":
+		case "civo", "digitalocean", "vultr", "k3s":
 			kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig)
 		case "google":
 			kcfg, err = clctrl.GoogleClient.GetContainerClusterAuth(clctrl.ClusterName, []byte(clctrl.GoogleAuth.KeyFile))
@@ -84,7 +84,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error {
 		switch clctrl.CloudProvider {
 		case "aws":
 			kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName)
-		case "civo", "digitalocean", "vultr":
+		case "civo", "digitalocean", "vultr", "k3s":
 			kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig)
 		case "google":
 			var err error
@@ -109,7 +109,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error {
 		var argoCDToken string
 
 		switch clctrl.CloudProvider {
-		case "aws", "civo", "google", "digitalocean", "vultr":
+		case "aws", "civo", "google", "digitalocean", "vultr", "k3s":
 
 			// kcfg.Clientset.RbacV1().
 			argoCDStopChannel := make(chan struct{}, 1)
@@ -163,7 +163,7 @@ func (clctrl *ClusterController) DeployRegistryApplication() error {
 		switch clctrl.CloudProvider {
 		case "aws":
 			kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName)
-		case "civo", "digitalocean", "vultr":
+		case "civo", "digitalocean", "vultr", "k3s":
 			kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig)
 		case "google":
 			var err error
@@ -207,6 +207,10 @@ func (clctrl *ClusterController) DeployRegistryApplication() error {
 			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 		} else if clctrl.CloudProvider == "vultr" && clctrl.GitProvider == "gitlab" {
 			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+		} else if clctrl.CloudProvider == "k3s" && clctrl.GitProvider == "github" {
+			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
+		} else if clctrl.CloudProvider == "k3s" && clctrl.GitProvider == "gitlab" {
+			registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName)
 		} else {
 			registryPath = fmt.Sprintf("registry/%s", clctrl.ClusterName)
 		}