From 47e55e57b6d948c43efe49846d7a04c8325eaaf8 Mon Sep 17 00:00:00 2001 From: DrummyFloyd Date: Tue, 31 Oct 2023 18:34:42 +0100 Subject: [PATCH] feat: add k3s backend Signed-off-by: DrummyFloyd --- extensions/k3s/env.go | 118 +++++++++++++ extensions/k3s/secrets.go | 110 ++++++++++++ internal/controller/argocd.go | 12 +- internal/controller/cluster.go | 47 +++-- internal/controller/controller.go | 12 +- internal/controller/git.go | 6 +- internal/controller/kubefirst.go | 2 +- internal/controller/repository.go | 50 ++++-- internal/controller/tools.go | 16 ++ internal/controller/users.go | 6 +- internal/controller/vault.go | 21 ++- internal/router/api/v1/cluster.go | 57 +++++- internal/router/api/v1/region.go | 4 + pkg/providerConfigs/adjustDriver.go | 88 +++++++++- pkg/providerConfigs/config.go | 5 + pkg/providerConfigs/detokenize.go | 38 +++- pkg/providerConfigs/types.go | 6 + pkg/types/auth.go | 8 + pkg/types/cluster.go | 15 +- providers/k3s/create.go | 261 ++++++++++++++++++++++++++++ providers/k3s/delete.go | 2 + 21 files changed, 814 insertions(+), 70 deletions(-) create mode 100644 extensions/k3s/env.go create mode 100644 extensions/k3s/secrets.go create mode 100644 providers/k3s/create.go create mode 100644 providers/k3s/delete.go diff --git a/extensions/k3s/env.go b/extensions/k3s/env.go new file mode 100644 index 00000000..c95a6826 --- /dev/null +++ b/extensions/k3s/env.go @@ -0,0 +1,118 @@ +/* +Copyright (C) 2021-2023, Kubefirst + +This program is licensed under MIT. +See the LICENSE file for more details. +*/ +package k3s + +import ( + "fmt" + "strconv" + "strings" + + "github.com/kubefirst/kubefirst-api/pkg/providerConfigs" + pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types" + "github.com/kubefirst/runtime/pkg/k8s" + "github.com/kubefirst/runtime/pkg/vault" + log "github.com/sirupsen/logrus" + "k8s.io/client-go/kubernetes" +) + +func readVaultTokenFromSecret(clientset *kubernetes.Clientset) string { + existingKubernetesSecret, err := k8s.ReadSecretV2(clientset, vault.VaultNamespace, vault.VaultSecretName) + if err != nil || existingKubernetesSecret == nil { + log.Printf("Error reading existing Secret data: %s", err) + return "" + } + + return existingKubernetesSecret["root-token"] +} + +func GetK3sTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string { + envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID + envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey + envs["AWS_SESSION_TOKEN"] = "" // allows for debugging + envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID + envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_aws_session_token"] = "" // allows for debugging + // envs["TF_LOG"] = "debug" + + return envs +} + +func GetGithubTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[string]string { + envs["GITHUB_TOKEN"] = cl.GitAuth.Token + envs["GITHUB_OWNER"] = cl.GitAuth.Owner + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID + envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID + envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey + envs["AWS_SESSION_TOKEN"] = "" // allows for debugging + envs["TF_VAR_aws_session_token"] = "" // allows for debugging + + return envs +} + +func GetGitlabTerraformEnvs(envs map[string]string, gid int, cl *pkgtypes.Cluster) map[string]string { + envs["GITLAB_TOKEN"] = cl.GitAuth.Token + envs["GITLAB_OWNER"] = cl.GitAuth.Owner + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID + envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID + envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_owner_group_id"] = strconv.Itoa(gid) + envs["TF_VAR_gitlab_owner"] = cl.GitAuth.Owner + envs["AWS_SESSION_TOKEN"] = "" // allows for debugging + envs["TF_VAR_aws_session_token"] = "" // allows for debugging + + return envs +} + +func GetUsersTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, envs map[string]string) map[string]string { + envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset) + envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL + envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token + envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner + envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID + envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID + envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey + envs["AWS_SESSION_TOKEN"] = "" // allows for debugging + envs["TF_VAR_aws_session_token"] = "" // allows for debugging + + return envs +} + +func GetVaultTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, envs map[string]string) map[string]string { + envs[fmt.Sprintf("%s_TOKEN", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Token + envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner + envs["TF_VAR_email_address"] = cl.AlertsEmail + envs["TF_VAR_vault_addr"] = providerConfigs.VaultPortForwardURL + envs["TF_VAR_vault_token"] = readVaultTokenFromSecret(clientset) + envs[fmt.Sprintf("TF_VAR_%s_token", cl.GitProvider)] = cl.GitAuth.Token + envs["VAULT_ADDR"] = providerConfigs.VaultPortForwardURL + envs["VAULT_TOKEN"] = readVaultTokenFromSecret(clientset) + envs["TF_VAR_atlantis_repo_webhook_secret"] = cl.AtlantisWebhookSecret + envs["TF_VAR_atlantis_repo_webhook_url"] = cl.AtlantisWebhookURL + envs["TF_VAR_kbot_ssh_private_key"] = cl.GitAuth.PrivateKey + envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey + envs["AWS_ACCESS_KEY_ID"] = cl.StateStoreCredentials.AccessKeyID + envs["AWS_SECRET_ACCESS_KEY"] = cl.StateStoreCredentials.SecretAccessKey + envs["TF_VAR_aws_access_key_id"] = cl.StateStoreCredentials.AccessKeyID + envs["TF_VAR_aws_secret_access_key"] = cl.StateStoreCredentials.SecretAccessKey + envs["AWS_SESSION_TOKEN"] = "" // allows for debugging + envs["TF_VAR_aws_session_token"] = "" // allows for debugging + + switch cl.GitProvider { + case "gitlab": + envs["TF_VAR_owner_group_id"] = fmt.Sprint(cl.GitlabOwnerGroupID) + } + + return envs +} diff --git a/extensions/k3s/secrets.go b/extensions/k3s/secrets.go new file mode 100644 index 00000000..b527b93f --- /dev/null +++ b/extensions/k3s/secrets.go @@ -0,0 +1,110 @@ +/* +Copyright (C) 2021-2023, Kubefirst + +This program is licensed under MIT. +See the LICENSE file for more details. +*/ +package k3s + +import ( + "context" + "strings" + + providerConfig "github.com/kubefirst/kubefirst-api/pkg/providerConfigs" + pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types" + "github.com/rs/zerolog/log" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func BootstrapK3sMgmtCluster(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster, destinationGitopsRepoURL string) error { + err := providerConfig.BootstrapMgmtCluster( + clientset, + cl.GitProvider, + cl.GitAuth.User, + destinationGitopsRepoURL, + cl.GitProtocol, + cl.CloudflareAuth.Token, + "", + cl.DnsProvider, + cl.CloudProvider, + cl.GitAuth.Token, + cl.GitAuth.PrivateKey, + ) + if err != nil { + log.Fatal().Msgf("error in central function to create secrets: %s", err) + return err + } + + var externalDnsToken string + switch cl.DnsProvider { + case "civo": + externalDnsToken = cl.CivoAuth.Token + case "vultr": + externalDnsToken = cl.VultrAuth.Token + case "digitalocean": + externalDnsToken = cl.DigitaloceanAuth.Token + case "aws": + externalDnsToken = "implement with cluster management" + case "google": + externalDnsToken = "implement with cluster management" + case "cloudflare": + externalDnsToken = cl.CloudflareAuth.APIToken + } + + // Create secrets + createSecrets := []*v1.Secret{ + { + ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "argo"}, + Data: map[string][]byte{ + "origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "atlantis"}, + Data: map[string][]byte{ + "origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "chartmuseum"}, + Data: map[string][]byte{ + "origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "external-dns-secrets", Namespace: "external-dns"}, + Data: map[string][]byte{ + "token": []byte(externalDnsToken), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "kubefirst"}, + Data: map[string][]byte{ + "origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cloudflare-creds", Namespace: "vault"}, + Data: map[string][]byte{ + "origin-ca-api-key": []byte(cl.CloudflareAuth.OriginCaIssuerKey), + }, + }, + } + for _, secret := range createSecrets { + _, err := clientset.CoreV1().Secrets(secret.ObjectMeta.Namespace).Get(context.TODO(), secret.ObjectMeta.Name, metav1.GetOptions{}) + if err == nil { + log.Info().Msgf("kubernetes secret %s/%s already created - skipping", secret.Namespace, secret.Name) + } else if strings.Contains(err.Error(), "not found") { + _, err = clientset.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + log.Fatal().Msgf("error creating kubernetes secret %s/%s: %s", secret.Namespace, secret.Name, err) + } + log.Info().Msgf("created kubernetes secret: %s/%s", secret.Namespace, secret.Name) + } + } + + return nil +} diff --git a/internal/controller/argocd.go b/internal/controller/argocd.go index 9fb8c039..f4073608 100644 --- a/internal/controller/argocd.go +++ b/internal/controller/argocd.go @@ -44,7 +44,7 @@ func (clctrl *ClusterController) InstallArgoCD() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": kcfg, err = clctrl.GoogleClient.GetContainerClusterAuth(clctrl.ClusterName, []byte(clctrl.GoogleAuth.KeyFile)) @@ -103,7 +103,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -128,7 +128,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error { var argoCDToken string switch clctrl.CloudProvider { - case "aws", "civo", "google", "digitalocean", "vultr": + case "aws", "civo", "google", "digitalocean", "vultr", "k3s": // kcfg.Clientset.RbacV1(). argoCDStopChannel := make(chan struct{}, 1) @@ -182,7 +182,7 @@ func (clctrl *ClusterController) DeployRegistryApplication() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -226,6 +226,10 @@ func (clctrl *ClusterController) DeployRegistryApplication() error { registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName) } else if clctrl.CloudProvider == "vultr" && clctrl.GitProvider == "gitlab" { registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName) + } else if clctrl.CloudProvider == "k3s" && clctrl.GitProvider == "github" { + registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName) + } else if clctrl.CloudProvider == "k3s" && clctrl.GitProvider == "gitlab" { + registryPath = fmt.Sprintf("registry/clusters/%s", clctrl.ClusterName) } else { registryPath = fmt.Sprintf("registry/%s", clctrl.ClusterName) } diff --git a/internal/controller/cluster.go b/internal/controller/cluster.go index 70609c09..9a0ae2c8 100644 --- a/internal/controller/cluster.go +++ b/internal/controller/cluster.go @@ -17,6 +17,7 @@ import ( civoext "github.com/kubefirst/kubefirst-api/extensions/civo" digitaloceanext "github.com/kubefirst/kubefirst-api/extensions/digitalocean" googleext "github.com/kubefirst/kubefirst-api/extensions/google" + k3sext "github.com/kubefirst/kubefirst-api/extensions/k3s" terraformext "github.com/kubefirst/kubefirst-api/extensions/terraform" vultrext "github.com/kubefirst/kubefirst-api/extensions/vultr" "github.com/kubefirst/kubefirst-api/internal/constants" @@ -65,7 +66,7 @@ func (clctrl *ClusterController) CreateCluster() error { return err } tfEnvs["TF_VAR_aws_account_id"] = *iamCaller.Account - tfEnvs["TF_VAR_use_ecr"] = strconv.FormatBool(clctrl.ECR) //Flag out the ecr terraform + tfEnvs["TF_VAR_use_ecr"] = strconv.FormatBool(clctrl.ECR) // Flag out the ecr terraform err = clctrl.MdbCl.UpdateCluster(clctrl.ClusterName, "aws_account_id", *iamCaller.Account) if err != nil { @@ -79,6 +80,8 @@ func (clctrl *ClusterController) CreateCluster() error { tfEnvs = googleext.GetGoogleTerraformEnvs(tfEnvs, &cl) case "vultr": tfEnvs = vultrext.GetVultrTerraformEnvs(tfEnvs, &cl) + case "k3s": + tfEnvs = k3sext.GetK3sTerraformEnvs(tfEnvs, &cl) } err := terraformext.InitApplyAutoApprove(clctrl.ProviderConfig.TerraformClient, tfEntrypoint, tfEnvs) @@ -132,9 +135,9 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { fullDomainName = clctrl.DomainName } - //handle set gitops tokens/values + // handle set gitops tokens/values switch kind { - case "gitops": //repo name + case "gitops": // repo name var externalDNSProviderTokenEnvName, externalDNSProviderSecretKey string if clctrl.DnsProvider == "cloudflare" { @@ -183,8 +186,8 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { NodeType: clctrl.NodeType, NodeCount: clctrl.NodeCount, KubefirstVersion: env.KubefirstVersion, - Kubeconfig: clctrl.ProviderConfig.Kubeconfig, //AWS - KubeconfigPath: clctrl.ProviderConfig.Kubeconfig, //Not AWS + Kubeconfig: clctrl.ProviderConfig.Kubeconfig, // AWS + KubeconfigPath: clctrl.ProviderConfig.Kubeconfig, // Not AWS ArgoCDIngressURL: fmt.Sprintf("https://argocd.%s", fullDomainName), ArgoCDIngressNoHTTPSURL: fmt.Sprintf("argocd.%s", fullDomainName), @@ -229,7 +232,7 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { ContainerRegistryURL: fmt.Sprintf("%s/%s", clctrl.ContainerRegistryHost, clctrl.GitAuth.Owner), } - //Handle provider specific tokens + // Handle provider specific tokens switch clctrl.CloudProvider { case "vultr": gitopsTemplateTokens.StateStoreBucketHostname = cl.StateStoreDetails.Hostname @@ -237,7 +240,7 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { gitopsTemplateTokens.GoogleAuth = clctrl.GoogleAuth.KeyFile gitopsTemplateTokens.GoogleProject = clctrl.GoogleAuth.ProjectId gitopsTemplateTokens.GoogleUniqueness = strings.ToLower(randstr.String(5)) - gitopsTemplateTokens.ForceDestroy = strconv.FormatBool(true) //TODO make this optional + gitopsTemplateTokens.ForceDestroy = strconv.FormatBool(true) // TODO make this optional gitopsTemplateTokens.KubefirstArtifactsBucket = clctrl.KubefirstStateStoreBucketName gitopsTemplateTokens.VaultDataBucketName = fmt.Sprintf("%s-vault-data-%s", clctrl.GoogleAuth.ProjectId, clctrl.ClusterName) case "aws": @@ -246,7 +249,7 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { return err } - //to be added to general tokens struct + // to be added to general tokens struct gitopsTemplateTokens.AwsIamArnAccountRoot = fmt.Sprintf("arn:aws:iam::%s:root", *iamCaller.Account) gitopsTemplateTokens.AwsNodeCapacityType = "ON_DEMAND" // todo adopt cli flag gitopsTemplateTokens.AwsAccountID = *iamCaller.Account @@ -262,10 +265,16 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { // gitopsTemplateTokens.ContainerRegistryURL = fmt.Sprintf("%s/%s", clctrl.ContainerRegistryHost, clctrl.GitAuth.Owner) log.Infof("NOT using ECR but instead %s URL %s", clctrl.GitProvider, gitopsTemplateTokens.ContainerRegistryURL) } + case "k3s": + gitopsTemplateTokens.K3sServersPrivateIps = clctrl.K3sAuth.K3sServersPrivateIps + gitopsTemplateTokens.K3sServersPublicIps = clctrl.K3sAuth.K3sServersPublicIps + gitopsTemplateTokens.SshUser = clctrl.K3sAuth.K3sSshUser + gitopsTemplateTokens.SshPrivateKey = clctrl.K3sAuth.K3sSshPrivateKey + gitopsTemplateTokens.K3sServersArgs = clctrl.K3sAuth.K3sServersArgs } return gitopsTemplateTokens - case "metaphor": //repo name + case "metaphor": // repo name metaphorTemplateTokens := &providerConfigs.MetaphorTokenValues{ ClusterName: clctrl.ClusterName, CloudRegion: clctrl.CloudRegion, @@ -293,7 +302,7 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -304,7 +313,7 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { } clientSet := kcfg.Clientset - //create namespaces + // create namespaces err = providerConfigs.K8sNamespaces(clientSet) if err != nil { return err @@ -315,7 +324,7 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { return err } - //TODO Remove specific ext bootstrap functions. + // TODO Remove specific ext bootstrap functions. if !cl.ClusterSecretsCreatedCheck { switch clctrl.CloudProvider { case "aws": @@ -353,9 +362,15 @@ func (clctrl *ClusterController) ClusterSecretsBootstrap() error { log.Errorf("error adding kubernetes secrets for bootstrap: %s", err) return err } + case "k3s": + err := k3sext.BootstrapK3sMgmtCluster(clientSet, &cl, destinationGitopsRepoGitURL) + if err != nil { + log.Errorf("error adding kubernetes secrets for bootstrap: %s", err) + return err + } } - //create service accounts + // create service accounts var token string if (clctrl.CloudflareAuth != pkgtypes.CloudflareAuth{}) { token = clctrl.CloudflareAuth.APIToken @@ -399,7 +414,7 @@ func (clctrl *ClusterController) ContainerRegistryAuth() (string, error) { } return containerRegistryAuthToken, nil - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -444,7 +459,7 @@ func (clctrl *ClusterController) WaitForClusterReady() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -457,7 +472,7 @@ func (clctrl *ClusterController) WaitForClusterReady() error { var dnsDeployment *v1.Deployment var err error switch clctrl.CloudProvider { - case "aws", "civo", "digitalocean", "vultr": + case "aws", "civo", "digitalocean", "vultr", "k3s": dnsDeployment, err = k8s.ReturnDeploymentObject( kcfg.Clientset, "kubernetes.io/name", diff --git a/internal/controller/controller.go b/internal/controller/controller.go index ac706a02..9ca804fd 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -57,6 +57,7 @@ type ClusterController struct { GitAuth pkgtypes.GitAuth VaultAuth pkgtypes.VaultAuth GoogleAuth pkgtypes.GoogleAuth + K3sAuth pkgtypes.K3sAuth AwsAccessKeyID string AwsSecretAccessKey string NodeType string @@ -165,7 +166,7 @@ func (clctrl *ClusterController) InitController(def *pkgtypes.ClusterDefinition) } clctrl.TelemetryEvent = telemetryEvent - //Copy Cluster Definiion to Cluster Controller + // Copy Cluster Definiion to Cluster Controller clctrl.AlertsEmail = def.AdminEmail clctrl.CloudProvider = def.CloudProvider clctrl.CloudRegion = def.CloudRegion @@ -184,6 +185,7 @@ func (clctrl *ClusterController) InitController(def *pkgtypes.ClusterDefinition) clctrl.DigitaloceanAuth = def.DigitaloceanAuth clctrl.VultrAuth = def.VultrAuth clctrl.GoogleAuth = def.GoogleAuth + clctrl.K3sAuth = def.K3sAuth clctrl.CloudflareAuth = def.CloudflareAuth clctrl.Repositories = []string{"gitops", "metaphor"} @@ -251,6 +253,13 @@ func (clctrl *ClusterController) InitController(def *pkgtypes.ClusterDefinition) case "vultr": clctrl.ProviderConfig = *providerConfigs.GetConfig(clctrl.ClusterName, clctrl.DomainName, clctrl.GitProvider, clctrl.GitAuth.Owner, clctrl.GitProtocol, clctrl.CloudflareAuth.Token, "") clctrl.ProviderConfig.VultrToken = clctrl.VultrAuth.Token + case "k3s": + clctrl.ProviderConfig = *providerConfigs.GetConfig(clctrl.ClusterName, clctrl.DomainName, clctrl.GitProvider, clctrl.GitAuth.Owner, clctrl.GitProtocol, clctrl.CloudflareAuth.Token, "") + clctrl.ProviderConfig.K3sServersPrivateIps = clctrl.K3sAuth.K3sServersPrivateIps + clctrl.ProviderConfig.K3sServersPublicIps = clctrl.K3sAuth.K3sServersPublicIps + clctrl.ProviderConfig.K3sSshPrivateKey = clctrl.K3sAuth.K3sSshPrivateKey + clctrl.ProviderConfig.K3sSshUser = clctrl.K3sAuth.K3sSshUser + clctrl.ProviderConfig.K3sServersArgs = clctrl.K3sAuth.K3sServersArgs } // Instantiate provider clients and copy cluster controller to cluster type @@ -303,6 +312,7 @@ func (clctrl *ClusterController) InitController(def *pkgtypes.ClusterDefinition) GoogleAuth: clctrl.GoogleAuth, DigitaloceanAuth: clctrl.DigitaloceanAuth, VultrAuth: clctrl.VultrAuth, + K3sAuth: clctrl.K3sAuth, CloudflareAuth: clctrl.CloudflareAuth, NodeType: clctrl.NodeType, NodeCount: clctrl.NodeCount, diff --git a/internal/controller/git.go b/internal/controller/git.go index 72b6200d..76cdd794 100644 --- a/internal/controller/git.go +++ b/internal/controller/git.go @@ -15,6 +15,7 @@ import ( civoext "github.com/kubefirst/kubefirst-api/extensions/civo" digitaloceanext "github.com/kubefirst/kubefirst-api/extensions/digitalocean" googleext "github.com/kubefirst/kubefirst-api/extensions/google" + k3sext "github.com/kubefirst/kubefirst-api/extensions/k3s" terraformext "github.com/kubefirst/kubefirst-api/extensions/terraform" vultrext "github.com/kubefirst/kubefirst-api/extensions/vultr" gitShim "github.com/kubefirst/kubefirst-api/internal/gitShim" @@ -95,6 +96,8 @@ func (clctrl *ClusterController) RunGitTerraform() error { tfEnvs = digitaloceanext.GetGithubTerraformEnvs(tfEnvs, &cl) case "vultr": tfEnvs = vultrext.GetGithubTerraformEnvs(tfEnvs, &cl) + case "k3s": + tfEnvs = k3sext.GetGithubTerraformEnvs(tfEnvs, &cl) } case "gitlab": switch clctrl.CloudProvider { @@ -108,6 +111,8 @@ func (clctrl *ClusterController) RunGitTerraform() error { tfEnvs = digitaloceanext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, &cl) case "vultr": tfEnvs = vultrext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, &cl) + case "k3s": + tfEnvs = k3sext.GetGitlabTerraformEnvs(tfEnvs, clctrl.GitlabOwnerGroupID, &cl) } } @@ -138,7 +143,6 @@ func (clctrl *ClusterController) RunGitTerraform() error { } func (clctrl *ClusterController) GetRepoURL() (string, error) { - // default case is https destinationGitopsRepoURL := clctrl.ProviderConfig.DestinationGitopsRepoURL diff --git a/internal/controller/kubefirst.go b/internal/controller/kubefirst.go index 4c224920..b203d407 100644 --- a/internal/controller/kubefirst.go +++ b/internal/controller/kubefirst.go @@ -49,7 +49,7 @@ func (clctrl *ClusterController) ExportClusterRecord() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error diff --git a/internal/controller/repository.go b/internal/controller/repository.go index bcac06b8..6996c19b 100644 --- a/internal/controller/repository.go +++ b/internal/controller/repository.go @@ -39,12 +39,12 @@ func (clctrl *ClusterController) RepositoryPrep() error { return err } - var useCloudflareOriginIssuer = false + useCloudflareOriginIssuer := false if cl.CloudflareAuth.OriginCaIssuerKey != "" { useCloudflareOriginIssuer = true } - //TODO Implement an interface so we can call GetDomainApexContent on the clustercotroller + // TODO Implement an interface so we can call GetDomainApexContent on the clustercotroller if !cl.GitopsReadyCheck { log.Info("initializing the gitops repository - this may take several minutes") @@ -62,9 +62,9 @@ func (clctrl *ClusterController) RepositoryPrep() error { clctrl.GitopsTemplateURL, clctrl.ProviderConfig.DestinationMetaphorRepoURL, clctrl.ProviderConfig.K1Dir, - clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), //tokens created on the fly + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly clctrl.ProviderConfig.MetaphorDir, - clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), //tokens created on the fly + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly true, cl.GitProtocol, useCloudflareOriginIssuer, @@ -84,9 +84,9 @@ func (clctrl *ClusterController) RepositoryPrep() error { clctrl.GitopsTemplateURL, clctrl.ProviderConfig.DestinationMetaphorRepoURL, clctrl.ProviderConfig.K1Dir, - clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), //tokens created on the fly + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly clctrl.ProviderConfig.MetaphorDir, - clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), //tokens created on the fly + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly civo.GetDomainApexContent(clctrl.DomainName), cl.GitProtocol, useCloudflareOriginIssuer, @@ -106,9 +106,9 @@ func (clctrl *ClusterController) RepositoryPrep() error { clctrl.GitopsTemplateURL, clctrl.ProviderConfig.DestinationMetaphorRepoURL, clctrl.ProviderConfig.K1Dir, - clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), //tokens created on the fly + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly clctrl.ProviderConfig.MetaphorDir, - clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), //tokens created on the fly + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly google.GetDomainApexContent(clctrl.DomainName), cl.GitProtocol, useCloudflareOriginIssuer, @@ -128,9 +128,9 @@ func (clctrl *ClusterController) RepositoryPrep() error { clctrl.GitopsTemplateURL, clctrl.ProviderConfig.DestinationMetaphorRepoURL, clctrl.ProviderConfig.K1Dir, - clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), //tokens created on the fly + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly clctrl.ProviderConfig.MetaphorDir, - clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), //tokens created on the fly + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly digitalocean.GetDomainApexContent(clctrl.DomainName), cl.GitProtocol, useCloudflareOriginIssuer, @@ -150,9 +150,9 @@ func (clctrl *ClusterController) RepositoryPrep() error { clctrl.GitopsTemplateURL, clctrl.ProviderConfig.DestinationMetaphorRepoURL, clctrl.ProviderConfig.K1Dir, - clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), //tokens created on the fly + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly clctrl.ProviderConfig.MetaphorDir, - clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), //tokens created on the fly + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly vultr.GetDomainApexContent(clctrl.DomainName), cl.GitProtocol, useCloudflareOriginIssuer, @@ -160,8 +160,30 @@ func (clctrl *ClusterController) RepositoryPrep() error { if err != nil { return err } - } + case "k3s": + err = providerConfigs.PrepareGitRepositories( + clctrl.CloudProvider, + clctrl.GitProvider, + clctrl.ClusterName, + clctrl.ClusterType, + clctrl.ProviderConfig.DestinationGitopsRepoURL, + clctrl.ProviderConfig.GitopsDir, + clctrl.GitopsTemplateBranch, + clctrl.GitopsTemplateURL, + clctrl.ProviderConfig.DestinationMetaphorRepoURL, + clctrl.ProviderConfig.K1Dir, + clctrl.CreateTokens("gitops").(*providerConfigs.GitopsDirectoryValues), // tokens created on the fly + clctrl.ProviderConfig.MetaphorDir, + clctrl.CreateTokens("metaphor").(*providerConfigs.MetaphorTokenValues), // tokens created on the fly + vultr.GetDomainApexContent(clctrl.DomainName), + cl.GitProtocol, + useCloudflareOriginIssuer, + ) + if err != nil { + return err + } + } err = clctrl.MdbCl.UpdateCluster(clctrl.ClusterName, "gitops_ready_check", true) if err != nil { return err @@ -217,7 +239,7 @@ func (clctrl *ClusterController) RepositoryPush() error { log.Errorf("unable to check for ssh keys in gitlab: %s", err.Error()) } - var keyName = "kbot-ssh-key" + keyName := "kbot-ssh-key" var keyFound bool = false for _, key := range keys { if key.Title == keyName { diff --git a/internal/controller/tools.go b/internal/controller/tools.go index 9fa08bfd..7acba866 100644 --- a/internal/controller/tools.go +++ b/internal/controller/tools.go @@ -102,6 +102,22 @@ func (clctrl *ClusterController) DownloadTools(toolsDir string) error { log.Errorf("error downloading dependencies: %s", err) return err } + + // TODO: move to runtime + // use vultr DownloadTools meanwhile + case "k3s": + err := vultr.DownloadTools( + clctrl.ProviderConfig.KubectlClient, + providerConfigs.KubectlClientVersion, + providerConfigs.LocalhostOS, + providerConfigs.LocalhostArch, + providerConfigs.TerraformClientVersion, + toolsDir, + ) + if err != nil { + log.Errorf("error downloading dependencies: %s", err) + return err + } } log.Info("dependency downloads complete") diff --git a/internal/controller/users.go b/internal/controller/users.go index cc9b7469..238e6a53 100644 --- a/internal/controller/users.go +++ b/internal/controller/users.go @@ -14,6 +14,7 @@ import ( civoext "github.com/kubefirst/kubefirst-api/extensions/civo" digitaloceanext "github.com/kubefirst/kubefirst-api/extensions/digitalocean" googleext "github.com/kubefirst/kubefirst-api/extensions/google" + k3sext "github.com/kubefirst/kubefirst-api/extensions/k3s" terraformext "github.com/kubefirst/kubefirst-api/extensions/terraform" vultrext "github.com/kubefirst/kubefirst-api/extensions/vultr" "github.com/kubefirst/metrics-client/pkg/telemetry" @@ -43,7 +44,7 @@ func (clctrl *ClusterController) RunUsersTerraform() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -75,6 +76,9 @@ func (clctrl *ClusterController) RunUsersTerraform() error { case "vultr": tfEnvs = vultrext.GetVultrTerraformEnvs(tfEnvs, &cl) tfEnvs = vultrext.GetUsersTerraformEnvs(kcfg.Clientset, &cl, tfEnvs) + case "k3s": + tfEnvs = k3sext.GetK3sTerraformEnvs(tfEnvs, &cl) + tfEnvs = k3sext.GetUsersTerraformEnvs(kcfg.Clientset, &cl, tfEnvs) } tfEntrypoint = clctrl.ProviderConfig.GitopsDir + "/terraform/users" terraformClient = clctrl.ProviderConfig.TerraformClient diff --git a/internal/controller/vault.go b/internal/controller/vault.go index 68a23a2a..79919760 100644 --- a/internal/controller/vault.go +++ b/internal/controller/vault.go @@ -21,6 +21,7 @@ import ( civoext "github.com/kubefirst/kubefirst-api/extensions/civo" digitaloceanext "github.com/kubefirst/kubefirst-api/extensions/digitalocean" googleext "github.com/kubefirst/kubefirst-api/extensions/google" + k3sext "github.com/kubefirst/kubefirst-api/extensions/k3s" terraformext "github.com/kubefirst/kubefirst-api/extensions/terraform" vultrext "github.com/kubefirst/kubefirst-api/extensions/vultr" "github.com/kubefirst/metrics-client/pkg/telemetry" @@ -49,7 +50,7 @@ func (clctrl *ClusterController) GetUserPassword(user string) error { // empty conf vaultConf := &vault.Conf - //sets up vault client within function + // sets up vault client within function clctrl.VaultAuth.KbotPassword, err = vaultConf.GetUserPassword(vault.VaultDefaultAddress, cl.VaultAuth.RootToken, "kbot", "initial-password") if err != nil { return err @@ -86,7 +87,7 @@ func (clctrl *ClusterController) InitializeVault() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) vaultHandlerPath = "github.com:kubefirst/manifests.git/vault-handler/replicas-3" case "google": @@ -127,7 +128,7 @@ func (clctrl *ClusterController) InitializeVault() error { if err != nil { return err } - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": // Initialize and unseal Vault // Build and apply manifests yamlData, err := kcfg.KustomizeBuild(vaultHandlerPath) @@ -188,7 +189,7 @@ func (clctrl *ClusterController) RunVaultTerraform() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -202,7 +203,7 @@ func (clctrl *ClusterController) RunVaultTerraform() error { tfEnvs := map[string]string{} - //Common TfEnvs + // Common TfEnvs var usernamePasswordString, base64DockerAuth, registryAuth string if clctrl.GitProvider == "gitlab" { @@ -225,7 +226,7 @@ func (clctrl *ClusterController) RunVaultTerraform() error { tfEnvs["TF_VAR_owner_group_id"] = strconv.Itoa(clctrl.GitlabOwnerGroupID) } - //Specific TfEnvs + // Specific TfEnvs switch clctrl.CloudProvider { case "aws": tfEnvs = awsext.GetVaultTerraformEnvs(kcfg.Clientset, &cl, tfEnvs) @@ -243,6 +244,9 @@ func (clctrl *ClusterController) RunVaultTerraform() error { case "vultr": tfEnvs = vultrext.GetVaultTerraformEnvs(kcfg.Clientset, &cl, tfEnvs) tfEnvs = vultrext.GetVultrTerraformEnvs(tfEnvs, &cl) + case "k3s": + tfEnvs = k3sext.GetVaultTerraformEnvs(kcfg.Clientset, &cl, tfEnvs) + tfEnvs = k3sext.GetK3sTerraformEnvs(tfEnvs, &cl) } tfEntrypoint := clctrl.ProviderConfig.GitopsDir + "/terraform/vault" @@ -319,7 +323,7 @@ func (clctrl *ClusterController) WriteVaultSecrets() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -384,7 +388,7 @@ func (clctrl *ClusterController) WaitForVault() error { switch clctrl.CloudProvider { case "aws": kcfg = awsext.CreateEKSKubeconfig(&clctrl.AwsClient.Config, clctrl.ClusterName) - case "civo", "digitalocean", "vultr": + case "civo", "digitalocean", "vultr", "k3s": kcfg = k8s.CreateKubeConfig(false, clctrl.ProviderConfig.Kubeconfig) case "google": var err error @@ -415,7 +419,6 @@ func (clctrl *ClusterController) WaitForVault() error { } func writeGoogleSecrets(homeDir string, vaultClient *vaultapi.Client) error { - // vault path - gcp/application-default-credentials adcJSON, err := os.ReadFile(fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)) if err != nil { diff --git a/internal/router/api/v1/cluster.go b/internal/router/api/v1/cluster.go index 46317464..85b9ce09 100644 --- a/internal/router/api/v1/cluster.go +++ b/internal/router/api/v1/cluster.go @@ -9,8 +9,8 @@ package api import ( "context" "fmt" - "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/kubefirst/kubefirst-api/internal/constants" @@ -25,6 +25,7 @@ import ( "github.com/kubefirst/kubefirst-api/providers/civo" "github.com/kubefirst/kubefirst-api/providers/digitalocean" "github.com/kubefirst/kubefirst-api/providers/google" + "github.com/kubefirst/kubefirst-api/providers/k3s" "github.com/kubefirst/kubefirst-api/providers/vultr" "github.com/kubefirst/metrics-client/pkg/telemetry" civoruntime "github.com/kubefirst/runtime/pkg/civo" @@ -272,20 +273,21 @@ func PostCreateCluster(c *gin.Context) { } } - //Retry mechanism + // Retry mechanism if cluster.ClusterName != "" { - //Assign cloud and git credentials + // Assign cloud and git credentials clusterDefinition.AWSAuth = cluster.AWSAuth clusterDefinition.CivoAuth = cluster.CivoAuth clusterDefinition.VultrAuth = cluster.VultrAuth clusterDefinition.DigitaloceanAuth = cluster.DigitaloceanAuth clusterDefinition.GoogleAuth = cluster.GoogleAuth + clusterDefinition.K3sAuth = cluster.K3sAuth clusterDefinition.GitAuth = cluster.GitAuth } // Determine authentication type useSecretForAuth := false - var k1AuthSecret = map[string]string{} + k1AuthSecret := map[string]string{} env, _ := env.GetEnv(constants.SilenceGetEnv) @@ -468,6 +470,50 @@ func PostCreateCluster(c *gin.Context) { } }() + c.JSON(http.StatusAccepted, types.JSONSuccessResponse{ + Message: "cluster create enqueued", + }) + // } + case "k3s": + if useSecretForAuth { + err := utils.ValidateAuthenticationFields(k1AuthSecret) + if err != nil { + c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ + Message: fmt.Sprintf("error checking k3s auth: %s", err), + }) + return + } + // force empty array if not server spubilc ips provided, to avoid errror of terraform tokenisation + defaultK3sServersPublicIps := []string{} + if k1AuthSecret["servers-public-ips"] != "" { + defaultK3sServersPublicIps = strings.Split(k1AuthSecret["servers-public-ips"], ",") + } + + clusterDefinition.K3sAuth = pkgtypes.K3sAuth{ + K3sServersPrivateIps: strings.Split(k1AuthSecret["servers-private-ips"], ","), + K3sServersPublicIps: defaultK3sServersPublicIps, + K3sSshUser: k1AuthSecret["ssh-user"], + K3sSshPrivateKey: k1AuthSecret["ssh-privatekey"], + K3sServersArgs: strings.Split(k1AuthSecret["servers-args"], ","), + } + } else { + if len(clusterDefinition.K3sAuth.K3sServersPrivateIps) == 0 || + clusterDefinition.K3sAuth.K3sSshUser == "" || + clusterDefinition.K3sAuth.K3sSshPrivateKey == "" { + c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ + // Message: "missing authentication credentials in request, please check and try again", + Message: fmt.Sprintf("missing authentication credentials in request, please check and try again: %v", clusterDefinition.K3sAuth), + }) + return + } + } + go func() { + err = k3s.CreateK3sCluster(&clusterDefinition) + if err != nil { + log.Errorf(err.Error()) + } + }() + c.JSON(http.StatusAccepted, types.JSONSuccessResponse{ Message: "cluster create enqueued", }) @@ -558,7 +604,6 @@ func GetClusterKubeconfig(c *gin.Context) { } kubeConfig, err := digitaloceanConf.GetKubeconfig(clusterName) - if err != nil { c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ Message: err.Error(), @@ -576,7 +621,6 @@ func GetClusterKubeconfig(c *gin.Context) { } kubeConfig, err := vultrConf.GetKubeconfig(clusterName) - if err != nil { c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ Message: err.Error(), @@ -592,6 +636,7 @@ func GetClusterKubeconfig(c *gin.Context) { }) return } + return } // PostImportCluster godoc diff --git a/internal/router/api/v1/region.go b/internal/router/api/v1/region.go index 75309446..59216d15 100644 --- a/internal/router/api/v1/region.go +++ b/internal/router/api/v1/region.go @@ -162,6 +162,10 @@ func PostRegions(c *gin.Context) { return } regionListResponse.Regions = regions + + case "k3s": + regionListResponse.Regions = []string{"on-premise (compatibilty-mode)"} + default: c.JSON(http.StatusBadRequest, types.JSONFailureResponse{ Message: fmt.Sprintf("unsupported provider: %s", cloudProvider), diff --git a/pkg/providerConfigs/adjustDriver.go b/pkg/providerConfigs/adjustDriver.go index 119f9f0e..e313bdc0 100644 --- a/pkg/providerConfigs/adjustDriver.go +++ b/pkg/providerConfigs/adjustDriver.go @@ -47,9 +47,8 @@ func AdjustGitopsRepo( } else if strings.Index(src, "/.terraform") > 0 { return true, nil } - //Add more stuff to be ignored here + // Add more stuff to be ignored here return false, nil - }, } @@ -443,6 +442,80 @@ func AdjustGitopsRepo( return nil } + K3S_GITLAB := "k3s-gitlab" + + if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == K3S_GITLAB { + driverContent := fmt.Sprintf("%s/%s-%s/", gitopsRepoDir, cloudProvider, gitProvider) + err := cp.Copy(driverContent, gitopsRepoDir, opt) + if err != nil { + log.Info().Msgf("Error populating gitops repository with driver content: %s. error: %s", fmt.Sprintf("%s-%s", cloudProvider, gitProvider), err.Error()) + return err + } + os.RemoveAll(driverContent) + + //* copy $HOME/.k1/gitops/templates/${clusterType}/* $HOME/.k1/gitops/registry/${clusterName} + clusterContent := fmt.Sprintf("%s/templates/%s", gitopsRepoDir, clusterType) + + // Remove apex content if apex content already exists + if apexContentExists { + log.Warn().Msgf("removing nginx-apex since apexContentExists was %v", apexContentExists) + os.Remove(fmt.Sprintf("%s/nginx-apex.yaml", clusterContent)) + os.RemoveAll(fmt.Sprintf("%s/nginx-apex", clusterContent)) + } else { + log.Warn().Msgf("will create nginx-apex since apexContentExists was %v", apexContentExists) + } + + if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == K3S_GITLAB { + err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/clusters/%s", gitopsRepoDir, clusterName), opt) + } else { + err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/%s", gitopsRepoDir, clusterName), opt) + } + if err != nil { + log.Info().Msgf("Error populating cluster content with %s. error: %s", clusterContent, err.Error()) + return err + } + os.RemoveAll(fmt.Sprintf("%s/templates/mgmt", gitopsRepoDir)) + + return nil + } + + K3S_GITHUB := "k3s-github" + + if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == K3S_GITHUB { + driverContent := fmt.Sprintf("%s/%s-%s/", gitopsRepoDir, cloudProvider, gitProvider) + err := cp.Copy(driverContent, gitopsRepoDir, opt) + if err != nil { + log.Info().Msgf("Error populating gitops repository with driver content: %s. error: %s", fmt.Sprintf("%s-%s", cloudProvider, gitProvider), err.Error()) + return err + } + os.RemoveAll(driverContent) + + //* copy $HOME/.k1/gitops/templates/${clusterType}/* $HOME/.k1/gitops/registry/${clusterName} + clusterContent := fmt.Sprintf("%s/templates/%s", gitopsRepoDir, clusterType) + + // Remove apex content if apex content already exists + if apexContentExists { + log.Warn().Msgf("removing nginx-apex since apexContentExists was %v", apexContentExists) + os.Remove(fmt.Sprintf("%s/nginx-apex.yaml", clusterContent)) + os.RemoveAll(fmt.Sprintf("%s/nginx-apex", clusterContent)) + } else { + log.Warn().Msgf("will create nginx-apex since apexContentExists was %v", apexContentExists) + } + + if strings.ToLower(fmt.Sprintf("%s-%s", cloudProvider, gitProvider)) == K3S_GITHUB { + err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/clusters/%s", gitopsRepoDir, clusterName), opt) + } else { + err = cp.Copy(clusterContent, fmt.Sprintf("%s/registry/%s", gitopsRepoDir, clusterName), opt) + } + if err != nil { + log.Info().Msgf("Error populating cluster content with %s. error: %s", clusterContent, err.Error()) + return err + } + os.RemoveAll(fmt.Sprintf("%s/templates/mgmt", gitopsRepoDir)) + + return nil + } + //* copy $cloudProvider-$gitProvider/* $HOME/.k1/gitops/ driverContent := fmt.Sprintf("%s/%s-%s/", gitopsRepoDir, cloudProvider, gitProvider) err := cp.Copy(driverContent, gitopsRepoDir, opt) @@ -504,9 +577,8 @@ func AdjustMetaphorRepo( } else if strings.Index(src, "/.terraform") > 0 { return true, nil } - //Add more stuff to be ignored here + // Add more stuff to be ignored here return false, nil - }, } @@ -517,7 +589,7 @@ func AdjustMetaphorRepo( os.RemoveAll(metaphorDir + "/.github") } - //todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from + // todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from // gitops and then allows gitops to commit during its sequence of ops if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) == AWS_GITHUB { //* metaphor app source @@ -567,7 +639,7 @@ func AdjustMetaphorRepo( os.RemoveAll(metaphorDir + "/.github") } - //todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from + // todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from // gitops and then allows gitops to commit during its sequence of ops if strings.ToLower(fmt.Sprintf("aws-%s", gitProvider)) == AWS_GITLAB { //* metaphor app source @@ -617,7 +689,7 @@ func AdjustMetaphorRepo( os.RemoveAll(metaphorDir + "/.github") } - //todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from + // todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from // gitops and then allows gitops to commit during its sequence of ops if strings.ToLower(fmt.Sprintf("civo-%s", gitProvider)) == CIVO_GITHUB { //* metaphor app source @@ -667,7 +739,7 @@ func AdjustMetaphorRepo( os.RemoveAll(metaphorDir + "/.github") } - //todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from + // todo implement repo, err :- createMetaphor() which returns the metaphor repoository object, removes content from // gitops and then allows gitops to commit during its sequence of ops if strings.ToLower(fmt.Sprintf("civo-%s", gitProvider)) == CIVO_GITLAB { //* metaphor app source diff --git a/pkg/providerConfigs/config.go b/pkg/providerConfigs/config.go index b12f7715..8c16e824 100644 --- a/pkg/providerConfigs/config.go +++ b/pkg/providerConfigs/config.go @@ -18,6 +18,11 @@ type ProviderConfig struct { DigitaloceanToken string GoogleAuth string GoogleProject string + K3sServersPrivateIps []string + K3sServersPublicIps []string + K3sSshPrivateKey string + K3sServersArgs []string + K3sSshUser string VultrToken string CloudflareAPIToken string CloudflareOriginCaIssuerAPIToken string diff --git a/pkg/providerConfigs/detokenize.go b/pkg/providerConfigs/detokenize.go index 2300b788..7a661a7c 100644 --- a/pkg/providerConfigs/detokenize.go +++ b/pkg/providerConfigs/detokenize.go @@ -7,8 +7,10 @@ See the LICENSE file for more details. package providerConfigs import ( + "encoding/json" "fmt" "io/ioutil" + "log" "os" "path/filepath" "strconv" @@ -27,7 +29,6 @@ func DetokenizeGitGitops(path string, tokens *GitopsDirectoryValues, gitProtocol func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol string, useCloudflareOriginIssuer bool) filepath.WalkFunc { return filepath.WalkFunc(func(path string, fi os.FileInfo, err error) error { - if fi.IsDir() && fi.Name() == ".git" { return filepath.SkipDir } @@ -96,6 +97,37 @@ func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol st newContents = strings.Replace(newContents, "", tokens.ForceDestroy, -1) newContents = strings.Replace(newContents, "", tokens.GoogleUniqueness, -1) + // k3s + newContents = strings.Replace(newContents, "", tokens.K3sServersPrivateIps[0], -1) + // TODO: this is a hack to get around + // need to be refactored into a single function with args + var terraformServersPrivateIpsList string + jsonBytes, err := json.Marshal(tokens.K3sServersPrivateIps) + if err != nil { + log.Fatalf("detokenise issue on %s", err) + } + terraformServersPrivateIpsList = string(jsonBytes) + newContents = strings.Replace(newContents, "", terraformServersPrivateIpsList, -1) + + var terraformServersPublicIpsList string + jsonBytes2, err := json.Marshal(tokens.K3sServersPublicIps) + if err != nil { + log.Fatalf("detokenise issue on %s", err) + } + terraformServersPublicIpsList = string(jsonBytes2) + newContents = strings.Replace(newContents, "", terraformServersPublicIpsList, -1) + + var terraformServersArgsList string + jsonBytes3, err := json.Marshal(tokens.K3sServersArgs) + if err != nil { + log.Fatalf("detokenise issue on %s", err) + } + terraformServersArgsList = string(jsonBytes3) + newContents = strings.Replace(newContents, "", terraformServersArgsList, -1) + + newContents = strings.Replace(newContents, "", tokens.SshUser, -1) + newContents = strings.Replace(newContents, "", tokens.SshPrivateKey, -1) + newContents = strings.Replace(newContents, "", tokens.ArgoCDIngressURL, -1) newContents = strings.Replace(newContents, "", tokens.ArgoCDIngressNoHTTPSURL, -1) newContents = strings.Replace(newContents, "", tokens.ArgoWorkflowsIngressURL, -1) @@ -143,7 +175,7 @@ func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol st newContents = strings.Replace(newContents, "", tokens.ExternalDNSProviderSecretKey, -1) newContents = strings.Replace(newContents, "", tokens.DomainName, -1) - //origin issuer defines which annotations should be on ingresses + // origin issuer defines which annotations should be on ingresses if useCloudflareOriginIssuer { newContents = strings.Replace(newContents, "", "cert-manager.io/issuer: cloudflare-origin-issuer", -1) newContents = strings.Replace(newContents, "", "cert-manager.io/issuer-kind: OriginIssuer", -1) @@ -161,7 +193,7 @@ func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol st // Switch the repo url based on https flag newContents = strings.Replace(newContents, "", tokens.GitopsRepoURL, -1) - //The fqdn is used by metaphor/argo to choose the appropriate url for cicd operations. + // The fqdn is used by metaphor/argo to choose the appropriate url for cicd operations. if gitProtocol == "https" { newContents = strings.Replace(newContents, "", fmt.Sprintf("https://%v.com/", tokens.GitProvider), -1) } else { diff --git a/pkg/providerConfigs/types.go b/pkg/providerConfigs/types.go index 41201304..fc72f0c8 100644 --- a/pkg/providerConfigs/types.go +++ b/pkg/providerConfigs/types.go @@ -51,6 +51,12 @@ type GitopsDirectoryValues struct { GoogleUniqueness string ForceDestroy string + K3sServersPrivateIps []string + K3sServersPublicIps []string + K3sServersArgs []string + SshUser string + SshPrivateKey string + GitDescription string GitNamespace string GitProvider string diff --git a/pkg/types/auth.go b/pkg/types/auth.go index cb022582..0bfcc2b8 100644 --- a/pkg/types/auth.go +++ b/pkg/types/auth.go @@ -65,3 +65,11 @@ type GoogleAuth struct { KeyFile string `bson:"key_file,omitempty" json:"key_file,omitempty"` ProjectId string `bson:"project_id,omitempty" json:"project_id,omitempty"` } + +type K3sAuth struct { + K3sServersPrivateIps []string `bson:"servers_private_ips,omitempty" json:"servers_private_ips,omitempty"` + K3sServersPublicIps []string `bson:"servers_public_ips,omitempty" json:"servers_public_ips,omitempty"` + K3sServersArgs []string `bson:"servers_args,omitempty" json:"servers_args,omitempty"` + K3sSshUser string `bson:"ssh_user,omitempty" json:"ssh_user,omitempty"` + K3sSshPrivateKey string `bson:"ssh_privatekey,omitempty" json:"ssh_privatekey,omitempty"` +} diff --git a/pkg/types/cluster.go b/pkg/types/cluster.go index a096b8e9..4d81a579 100644 --- a/pkg/types/cluster.go +++ b/pkg/types/cluster.go @@ -12,10 +12,9 @@ import ( // ClusterDefinition describes an incoming request to create a cluster type ClusterDefinition struct { - - //Cluster + // Cluster AdminEmail string `json:"admin_email" binding:"required"` - CloudProvider string `json:"cloud_provider" binding:"required,oneof=aws civo digitalocean vultr google"` + CloudProvider string `json:"cloud_provider" binding:"required,oneof=aws civo digitalocean vultr google k3s"` CloudRegion string `json:"cloud_region" binding:"required"` ClusterName string `json:"cluster_name,omitempty"` DomainName string `json:"domain_name" binding:"required"` @@ -26,22 +25,25 @@ type ClusterDefinition struct { NodeType string `json:"node_type" binding:"required"` NodeCount int `json:"node_count" binding:"required"` - //Git + // Git + + // Git GitopsTemplateURL string `json:"gitops_template_url"` GitopsTemplateBranch string `json:"gitops_template_branch"` GitProvider string `json:"git_provider" binding:"required,oneof=github gitlab"` GitProtocol string `json:"git_protocol" binding:"required,oneof=ssh https"` - //AWS + // AWS ECR bool `json:"ecr,omitempty"` - //Auth + // Auth AWSAuth AWSAuth `json:"aws_auth,omitempty"` CivoAuth CivoAuth `json:"civo_auth,omitempty"` DigitaloceanAuth DigitaloceanAuth `json:"do_auth,omitempty"` VultrAuth VultrAuth `json:"vultr_auth,omitempty"` CloudflareAuth CloudflareAuth `json:"cloudflare_auth,omitempty"` GoogleAuth GoogleAuth `json:"google_auth,omitempty"` + K3sAuth K3sAuth `json:"k3s_auth,omitempty"` GitAuth GitAuth `json:"git_auth,omitempty"` } @@ -75,6 +77,7 @@ type Cluster struct { GitAuth GitAuth `bson:"git_auth,omitempty" json:"git_auth,omitempty"` VaultAuth VaultAuth `bson:"vault_auth,omitempty" json:"vault_auth,omitempty"` GoogleAuth GoogleAuth `bson:"google_auth,omitempty" json:"google_auth,omitempty"` + K3sAuth K3sAuth `bson:"k3s_auth,omitempty" json:"k3s_auth,omitempty"` GitopsTemplateURL string `bson:"gitops_template_url" json:"gitops_template_url"` GitopsTemplateBranch string `bson:"gitops_template_branch" json:"gitops_template_branch"` diff --git a/providers/k3s/create.go b/providers/k3s/create.go new file mode 100644 index 00000000..ab99b9dd --- /dev/null +++ b/providers/k3s/create.go @@ -0,0 +1,261 @@ +/* +Copyright (C) 2021-2023, Kubefirst + +This program is licensed under MIT. +See the LICENSE file for more details. +*/ +package k3s + +import ( + "os" + + "github.com/kubefirst/kubefirst-api/internal/constants" + "github.com/kubefirst/kubefirst-api/internal/controller" + "github.com/kubefirst/kubefirst-api/internal/db" + "github.com/kubefirst/kubefirst-api/internal/services" + pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types" + "github.com/kubefirst/runtime/pkg/k8s" + "github.com/kubefirst/runtime/pkg/ssl" + log "github.com/sirupsen/logrus" +) + +// Createk3sCluster +func CreateK3sCluster(definition *pkgtypes.ClusterDefinition) error { + ctrl := controller.ClusterController{} + err := ctrl.InitController(definition) + if err != nil { + return err + } + + err = ctrl.MdbCl.UpdateCluster(ctrl.ClusterName, "in_progress", true) + if err != nil { + return err + } + + err = ctrl.DownloadTools(ctrl.ProviderConfig.ToolsDir) + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.DomainLivenessTest() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.StateStoreCredentials() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.GitInit() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.InitializeBot() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.RepositoryPrep() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.RunGitTerraform() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.RepositoryPush() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.CreateCluster() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.WaitForClusterReady() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.ClusterSecretsBootstrap() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + //* check for ssl restore + log.Info("checking for tls secrets to restore") + secretsFilesToRestore, err := os.ReadDir(ctrl.ProviderConfig.SSLBackupDir + "/secrets") + if err != nil { + log.Infof("%s", err) + } + if len(secretsFilesToRestore) != 0 { + // todo would like these but requires CRD's and is not currently supported + // add crds ( use execShellReturnErrors? ) + // https://raw.githubusercontent.com/cert-manager/cert-manager/v1.11.0/deploy/crds/crd-clusterissuers.yaml + // https://raw.githubusercontent.com/cert-manager/cert-manager/v1.11.0/deploy/crds/crd-certificates.yaml + // add certificates, and clusterissuers + log.Infof("found %d tls secrets to restore", len(secretsFilesToRestore)) + ssl.Restore(ctrl.ProviderConfig.SSLBackupDir, ctrl.DomainName, ctrl.ProviderConfig.Kubeconfig) + } else { + log.Info("no files found in secrets directory, continuing") + } + + err = ctrl.InstallArgoCD() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.InitializeArgoCD() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.DeployRegistryApplication() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.WaitForVault() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.InitializeVault() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + // Create kubeconfig client + kcfg := k8s.CreateKubeConfig(false, ctrl.ProviderConfig.Kubeconfig) + + // SetupMinioStorage(kcfg, ctrl.ProviderConfig.K1Dir, ctrl.GitProvider) + + //* configure vault with terraform + //* vault port-forward + vaultStopChannel := make(chan struct{}, 1) + defer func() { + close(vaultStopChannel) + }() + k8s.OpenPortForwardPodWrapper( + kcfg.Clientset, + kcfg.RestConfig, + "vault-0", + "vault", + 8200, + 8200, + vaultStopChannel, + ) + + err = ctrl.RunVaultTerraform() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.WriteVaultSecrets() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + err = ctrl.RunUsersTerraform() + if err != nil { + ctrl.HandleError(err.Error()) + return err + } + + // Wait for last sync wave app transition to Running + log.Info("waiting for final sync wave Deployment to transition to Running") + crossplaneDeployment, err := k8s.ReturnDeploymentObject( + kcfg.Clientset, + "app.kubernetes.io/instance", + "crossplane", + "crossplane-system", + 3600, + ) + if err != nil { + log.Errorf("Error finding crossplane Deployment: %s", err) + ctrl.HandleError(err.Error()) + return err + } + log.Infof("waiting on dns, tls certificates from letsencrypt and remaining sync waves.\n this may take up to 60 minutes but regularly completes in under 20 minutes") + _, err = k8s.WaitForDeploymentReady(kcfg.Clientset, crossplaneDeployment, 3600) + if err != nil { + log.Errorf("Error waiting for all Apps to sync ready state: %s", err) + + ctrl.HandleError(err.Error()) + return err + } + + //* export and import cluster + err = ctrl.ExportClusterRecord() + if err != nil { + log.Errorf("Error exporting cluster record: %s", err) + return err + } else { + err = ctrl.MdbCl.UpdateCluster(ctrl.ClusterName, "status", constants.ClusterStatusProvisioned) + if err != nil { + return err + } + + err = ctrl.MdbCl.UpdateCluster(ctrl.ClusterName, "in_progress", false) + if err != nil { + return err + } + + log.Info("cluster creation complete") + + // Create default service entries + cl, _ := db.Client.GetCluster(ctrl.ClusterName) + err = services.AddDefaultServices(&cl) + if err != nil { + log.Errorf("error adding default service entries for cluster %s: %s", cl.ClusterName, err) + } + } + + log.Info("waiting for kubefirst-api Deployment to transition to Running") + kubefirstAPI, err := k8s.ReturnDeploymentObject( + kcfg.Clientset, + "app.kubernetes.io/name", + "kubefirst-api", + "kubefirst", + 1200, + ) + if err != nil { + log.Errorf("Error finding kubefirst api Deployment: %s", err) + ctrl.HandleError(err.Error()) + return err + } + _, err = k8s.WaitForDeploymentReady(kcfg.Clientset, kubefirstAPI, 300) + if err != nil { + log.Errorf("Error waiting for kubefirst-api to transition to Running: %s", err) + + ctrl.HandleError(err.Error()) + return err + } + + log.Info("cluster creation complete") + + return nil +} diff --git a/providers/k3s/delete.go b/providers/k3s/delete.go new file mode 100644 index 00000000..ec41e3b2 --- /dev/null +++ b/providers/k3s/delete.go @@ -0,0 +1,2 @@ +// TODO: +package k3s