Merge branch 'main' into add-k3s-new-cli

.github/ISSUE_TEMPLATE/bugs.yml

@@ -46,9 +46,9 @@ body:
       multiple: true
       options:
         - None specific
-        - UI (Console app)
         - CLI
-        - Civo Marketplace
+        - Marketplace
+        - UI (Console app)
     validations:
       required: true
   - type: dropdown

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021-2023 Kubefirst
+Copyright (c) 2021-2024 Kubefirst
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

cmd/aws/command.go

@@ -30,10 +30,12 @@ var (
 	gitopsTemplateURLFlag    string
 	gitopsTemplateBranchFlag string
 	domainNameFlag           string
+	subdomainNameFlag        string
 	useTelemetryFlag         bool
 	ecrFlag                  bool
 	nodeTypeFlag             string
 	nodeCountFlag            string
+	installCatalogApps       string
 
 	// RootCredentials
 	copyArgoCDPasswordToClipboardFlag bool
@@ -88,6 +90,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&nodeCountFlag, "node-count", awsDefaults.NodeCount, "the node count for the cluster")
 	createCmd.Flags().StringVar(&nodeTypeFlag, "node-type", awsDefaults.InstanceSize, "the instance size of the cluster to create")
 	createCmd.Flags().StringVar(&dnsProviderFlag, "dns-provider", "aws", fmt.Sprintf("the dns provider - one of: %s", supportedDNSProviders))
+	createCmd.Flags().StringVar(&subdomainNameFlag, "subdomain", "", "the subdomain to use for DNS records (Cloudflare)")
 	createCmd.Flags().StringVar(&domainNameFlag, "domain-name", "", "the Route53/Cloudflare hosted zone name to use for DNS records (i.e. your-domain.com|subdomain.your-domain.com) (required)")
 	createCmd.MarkFlagRequired("domain-name")
 	createCmd.Flags().StringVar(&gitProviderFlag, "git-provider", "github", fmt.Sprintf("the git provider - one of: %s", supportedGitProviders))
@@ -96,6 +99,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&gitlabGroupFlag, "gitlab-group", "", "the GitLab group for the new gitops and metaphor projects - required if using gitlab")
 	createCmd.Flags().StringVar(&gitopsTemplateBranchFlag, "gitops-template-branch", "", "the branch to clone for the gitops-template repository")
 	createCmd.Flags().StringVar(&gitopsTemplateURLFlag, "gitops-template-url", "https://github.com/kubefirst/gitops-template.git", "the fully qualified url to the gitops-template repository to clone")
+	createCmd.Flags().StringVar(&installCatalogApps, "install-catalog-apps", "", "comma seperated values to install after provision")
 	createCmd.Flags().BoolVar(&useTelemetryFlag, "use-telemetry", true, "whether to emit telemetry")
 	createCmd.Flags().BoolVar(&ecrFlag, "ecr", false, "whether or not to use ecr vs the git provider")
 

cmd/aws/create.go

@@ -11,6 +11,7 @@ import (
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/kubefirst/kubefirst/internal/catalog"
 	"github.com/kubefirst/kubefirst/internal/cluster"
 	"github.com/kubefirst/kubefirst/internal/gitShim"
 	"github.com/kubefirst/kubefirst/internal/launch"
@@ -34,6 +35,11 @@ func createAws(cmd *cobra.Command, args []string) error {
 
 	progress.DisplayLogHints(40)
 
+	isValid, catalogApps, err := catalog.ValidateCatalogApps(cliFlags.InstallCatalogApps)
+	if !isValid {
+		return err
+	}
+
 	err = ValidateProvidedFlags(cliFlags.GitProvider)
 	if err != nil {
 		progress.Error(err.Error())
@@ -112,7 +118,7 @@ func createAws(cmd *cobra.Command, args []string) error {
 		progress.Error("unable to start kubefirst api")
 	}
 
-	provision.CreateMgmtCluster(gitAuth, cliFlags)
+	provision.CreateMgmtCluster(gitAuth, cliFlags, catalogApps)
 
 	return nil
 }

cmd/civo/command.go

@@ -23,6 +23,7 @@ var (
 	clusterNameFlag          string
 	clusterTypeFlag          string
 	dnsProviderFlag          string
+	subdomainNameFlag        string
 	domainNameFlag           string
 	githubOrgFlag            string
 	gitlabGroupFlag          string
@@ -33,6 +34,7 @@ var (
 	useTelemetryFlag         bool
 	nodeTypeFlag             string
 	nodeCountFlag            string
+	installCatalogApps       string
 
 	// RootCredentials
 	copyArgoCDPasswordToClipboardFlag bool
@@ -99,6 +101,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&nodeCountFlag, "node-count", civoDefaults.NodeCount, "the node count for the cluster")
 	createCmd.Flags().StringVar(&nodeTypeFlag, "node-type", civoDefaults.InstanceSize, "the instance size of the cluster to create")
 	createCmd.Flags().StringVar(&dnsProviderFlag, "dns-provider", "civo", fmt.Sprintf("the dns provider - one of: %s", supportedDNSProviders))
+	createCmd.Flags().StringVar(&subdomainNameFlag, "subdomain", "", "the subdomain to use for DNS records (Cloudflare)")
 	createCmd.Flags().StringVar(&domainNameFlag, "domain-name", "", "the Civo DNS Name to use for DNS records (i.e. your-domain.com|subdomain.your-domain.com) (required)")
 	createCmd.MarkFlagRequired("domain-name")
 	createCmd.Flags().StringVar(&gitProviderFlag, "git-provider", "github", fmt.Sprintf("the git provider - one of: %s", supportedGitProviders))
@@ -107,6 +110,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&gitlabGroupFlag, "gitlab-group", "", "the GitLab group for the new gitops and metaphor projects - required if using gitlab")
 	createCmd.Flags().StringVar(&gitopsTemplateBranchFlag, "gitops-template-branch", "", "the branch to clone for the gitops-template repository")
 	createCmd.Flags().StringVar(&gitopsTemplateURLFlag, "gitops-template-url", "https://github.com/kubefirst/gitops-template.git", "the fully qualified url to the gitops-template repository to clone")
+	createCmd.Flags().StringVar(&installCatalogApps, "install-catalog-apps", "", "comma seperated values to install after provision")
 	createCmd.Flags().BoolVar(&useTelemetryFlag, "use-telemetry", true, "whether to emit telemetry")
 
 	return createCmd

cmd/civo/create.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/kubefirst/kubefirst/internal/catalog"
 	"github.com/kubefirst/kubefirst/internal/cluster"
 	"github.com/kubefirst/kubefirst/internal/gitShim"
 	"github.com/kubefirst/kubefirst/internal/launch"
@@ -32,19 +33,18 @@ func createCivo(cmd *cobra.Command, args []string) error {
 
 	progress.DisplayLogHints(15)
 
+	isValid, catalogApps, err := catalog.ValidateCatalogApps(cliFlags.InstallCatalogApps)
+	if !isValid {
+		return err
+	}
+
 	err = ValidateProvidedFlags(cliFlags.GitProvider)
 	if err != nil {
 		progress.Error(err.Error())
 		return nil
 	}
 
 	// If cluster setup is complete, return
-	clusterSetupComplete := viper.GetBool("kubefirst-checks.cluster-install-complete")
-	if clusterSetupComplete {
-		err = fmt.Errorf("this cluster install process has already completed successfully")
-		progress.Error(err.Error())
-		return nil
-	}
 
 	utilities.CreateK1ClusterDirectory(clusterNameFlag)
 
@@ -88,7 +88,7 @@ func createCivo(cmd *cobra.Command, args []string) error {
 		progress.Error("unable to start kubefirst api")
 	}
 
-	provision.CreateMgmtCluster(gitAuth, cliFlags)
+	provision.CreateMgmtCluster(gitAuth, cliFlags, catalogApps)
 
 	return nil
 }

cmd/digitalocean/command.go

@@ -25,6 +25,7 @@ var (
 	clusterTypeFlag          string
 	dnsProviderFlag          string
 	domainNameFlag           string
+	subdomainNameFlag        string
 	githubOrgFlag            string
 	gitlabGroupFlag          string
 	gitProviderFlag          string
@@ -34,6 +35,7 @@ var (
 	useTelemetryFlag         bool
 	nodeTypeFlag             string
 	nodeCountFlag            string
+	installCatalogApps       string
 
 	// RootCredentials
 	copyArgoCDPasswordToClipboardFlag bool
@@ -92,6 +94,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&nodeCountFlag, "node-count", doDefaults.NodeCount, "the node count for the cluster")
 	createCmd.Flags().StringVar(&nodeTypeFlag, "node-type", doDefaults.InstanceSize, "the instance size of the cluster to create")
 	createCmd.Flags().StringVar(&dnsProviderFlag, "dns-provider", "digitalocean", fmt.Sprintf("the dns provider - one of: %s", supportedDNSProviders))
+	createCmd.Flags().StringVar(&subdomainNameFlag, "subdomain", "", "the subdomain to use for DNS records (Cloudflare)")
 	createCmd.Flags().StringVar(&domainNameFlag, "domain-name", "", "the DigitalOcean DNS Name to use for DNS records (i.e. your-domain.com|subdomain.your-domain.com) (required)")
 	createCmd.MarkFlagRequired("domain-name")
 	createCmd.Flags().StringVar(&gitProviderFlag, "git-provider", "github", fmt.Sprintf("the git provider - one of: %s", supportedGitProviders))
@@ -100,6 +103,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&gitlabGroupFlag, "gitlab-group", "", "the GitLab group for the new gitops and metaphor projects - required if using gitlab")
 	createCmd.Flags().StringVar(&gitopsTemplateBranchFlag, "gitops-template-branch", "", "the branch to clone for the gitops-template repository")
 	createCmd.Flags().StringVar(&gitopsTemplateURLFlag, "gitops-template-url", "https://github.com/kubefirst/gitops-template.git", "the fully qualified url to the gitops-template repository to clone")
+	createCmd.Flags().StringVar(&installCatalogApps, "install-catalog-apps", "", "comma seperated values to install after provision")
 	createCmd.Flags().BoolVar(&useTelemetryFlag, "use-telemetry", true, "whether to emit telemetry")
 
 	return createCmd

cmd/digitalocean/create.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	"github.com/kubefirst/kubefirst/internal/catalog"
 	"github.com/kubefirst/kubefirst/internal/cluster"
 	"github.com/kubefirst/kubefirst/internal/gitShim"
 	"github.com/kubefirst/kubefirst/internal/launch"
@@ -33,6 +34,11 @@ func createDigitalocean(cmd *cobra.Command, args []string) error {
 
 	progress.DisplayLogHints(20)
 
+	isValid, catalogApps, err := catalog.ValidateCatalogApps(cliFlags.InstallCatalogApps)
+	if !isValid {
+		return err
+	}
+
 	err = ValidateProvidedFlags(cliFlags.GitProvider)
 	if err != nil {
 		progress.Error(err.Error())
@@ -89,7 +95,7 @@ func createDigitalocean(cmd *cobra.Command, args []string) error {
 		progress.Error("unable to start kubefirst api")
 	}
 
-	provision.CreateMgmtCluster(gitAuth, cliFlags)
+	provision.CreateMgmtCluster(gitAuth, cliFlags, catalogApps)
 
 	return nil
 }

cmd/google/command.go

@@ -24,6 +24,7 @@ var (
 	clusterTypeFlag          string
 	dnsProviderFlag          string
 	domainNameFlag           string
+	subdomainNameFlag        string
 	googleProjectFlag        string
 	githubOrgFlag            string
 	gitlabGroupFlag          string
@@ -35,6 +36,7 @@ var (
 	forceDestroyFlag         bool
 	nodeTypeFlag             string
 	nodeCountFlag            string
+	installCatalogApps       string
 
 	// RootCredentials
 	copyArgoCDPasswordToClipboardFlag bool
@@ -94,6 +96,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&nodeCountFlag, "node-count", googleDefaults.NodeCount, "the node count for the cluster")
 	createCmd.Flags().StringVar(&nodeTypeFlag, "node-type", googleDefaults.InstanceSize, "the instance size of the cluster to create")
 	createCmd.Flags().StringVar(&dnsProviderFlag, "dns-provider", "google", fmt.Sprintf("the dns provider - one of: %s", supportedDNSProviders))
+	createCmd.Flags().StringVar(&subdomainNameFlag, "subdomain", "", "the subdomain to use for DNS records (Cloudflare)")
 	createCmd.Flags().StringVar(&domainNameFlag, "domain-name", "", "the GCP DNS Name to use for DNS records (i.e. your-domain.com|subdomain.your-domain.com) (required)")
 	createCmd.MarkFlagRequired("domain-name")
 	createCmd.Flags().StringVar(&googleProjectFlag, "google-project", "", "google project id (required)")
@@ -104,6 +107,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&gitlabGroupFlag, "gitlab-group", "", "the GitLab group for the new gitops and metaphor projects - required if using gitlab")
 	createCmd.Flags().StringVar(&gitopsTemplateBranchFlag, "gitops-template-branch", "", "the branch to clone for the gitops-template repository")
 	createCmd.Flags().StringVar(&gitopsTemplateURLFlag, "gitops-template-url", "https://github.com/kubefirst/gitops-template.git", "the fully qualified url to the gitops-template repository to clone")
+	createCmd.Flags().StringVar(&installCatalogApps, "install-catalog-apps", "", "comma seperated values to install after provision")
 	createCmd.Flags().BoolVar(&useTelemetryFlag, "use-telemetry", true, "whether to emit telemetry")
 	createCmd.Flags().BoolVar(&forceDestroyFlag, "force-destroy", false, "allows force destruction on objects (helpful for test environments, defaults to false)")
 	return createCmd

cmd/google/create.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	"github.com/kubefirst/kubefirst/internal/catalog"
 	"github.com/kubefirst/kubefirst/internal/cluster"
 	"github.com/kubefirst/kubefirst/internal/gitShim"
 	"github.com/kubefirst/kubefirst/internal/launch"
@@ -34,6 +35,11 @@ func createGoogle(cmd *cobra.Command, args []string) error {
 
 	progress.DisplayLogHints(20)
 
+	isValid, catalogApps, err := catalog.ValidateCatalogApps(cliFlags.InstallCatalogApps)
+	if !isValid {
+		return err
+	}
+
 	err = ValidateProvidedFlags(cliFlags.GitProvider)
 	if err != nil {
 		progress.Error(err.Error())
@@ -87,7 +93,7 @@ func createGoogle(cmd *cobra.Command, args []string) error {
 		progress.Error("unable to start kubefirst api")
 	}
 
-	provision.CreateMgmtCluster(gitAuth, cliFlags)
+	provision.CreateMgmtCluster(gitAuth, cliFlags, catalogApps)
 
 	return nil
 }

cmd/k3d/command.go

@@ -29,6 +29,7 @@ var (
 	gitopsTemplateURLFlag    string
 	gitopsTemplateBranchFlag string
 	useTelemetryFlag         bool
+	installCatalogApps       string
 
 	// RootCredentials
 	copyArgoCDPasswordToClipboardFlag bool
@@ -95,6 +96,7 @@ func Create() *cobra.Command {
 	createCmd.Flags().StringVar(&gitlabGroupFlag, "gitlab-group", "", "the GitLab group for the new gitops and metaphor projects - required if using gitlab")
 	createCmd.Flags().StringVar(&gitopsTemplateBranchFlag, "gitops-template-branch", "", "the branch to clone for the gitops-template repository")
 	createCmd.Flags().StringVar(&gitopsTemplateURLFlag, "gitops-template-url", "https://github.com/kubefirst/gitops-template.git", "the fully qualified url to the gitops-template repository to clone")
+	createCmd.Flags().StringVar(&installCatalogApps, "install-catalog-apps", "", "comma seperated values of catalog apps to install after provision")
 	createCmd.Flags().BoolVar(&useTelemetryFlag, "use-telemetry", true, "whether to emit telemetry")
 
 	return createCmd

cmd/k3d/create.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kubefirst/kubefirst-api/pkg/handlers"
 	"github.com/kubefirst/kubefirst-api/pkg/reports"
 	"github.com/kubefirst/kubefirst-api/pkg/wrappers"
+	"github.com/kubefirst/kubefirst/internal/catalog"
 	"github.com/kubefirst/kubefirst/internal/gitShim"
 	"github.com/kubefirst/kubefirst/internal/segment"
 	"github.com/kubefirst/kubefirst/internal/utilities"
@@ -105,6 +106,11 @@ func runK3d(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	installCatalogAppsFlag, err := cmd.Flags().GetString("install-catalog-apps")
+	if err != nil {
+		return err
+	}
+
 	useTelemetryFlag, err := cmd.Flags().GetBool("use-telemetry")
 	if err != nil {
 		return err
@@ -119,6 +125,11 @@ func runK3d(cmd *cobra.Command, args []string) error {
 	utilities.CreateK1ClusterDirectory(clusterNameFlag)
 	helpers.DisplayLogHints()
 
+	isValid, catalogApps, err := catalog.ValidateCatalogApps(installCatalogAppsFlag)
+	if !isValid {
+		return err
+	}
+
 	switch gitProviderFlag {
 	case "github":
 		key, err := internalssh.GetHostKey("github.com")
@@ -369,6 +380,13 @@ func runK3d(cmd *cobra.Command, args []string) error {
 		viper.WriteConfig()
 	}
 
+	atlantisNgrokAuthtoken := viper.GetString("secrets.atlantis-ngrok-authtoken")
+	if atlantisNgrokAuthtoken == "" {
+		atlantisNgrokAuthtoken = os.Getenv("NGROK_AUTHTOKEN")
+		viper.Set("secrets.atlantis-ngrok-authtoken", atlantisNgrokAuthtoken)
+		viper.WriteConfig()
+	}
+
 	log.Info().Msg("checking authentication to required providers")
 
 	// check disk
@@ -544,6 +562,10 @@ func runK3d(cmd *cobra.Command, args []string) error {
 	progressPrinter.IncrementTracker("preflight-checks", 1)
 	progressPrinter.AddTracker("cloning-and-formatting-git-repositories", "Cloning and formatting git repositories", 1)
 	progressPrinter.SetupProgress(progressPrinter.TotalOfTrackers(), false)
+	removeAtlantis := false
+	if viper.GetString("secrets.atlantis-ngrok-authtoken") == "" {
+		removeAtlantis = true
+	}
 	if !viper.GetBool("kubefirst-checks.gitops-ready-to-push") {
 		log.Info().Msg("generating your new gitops repository")
 		err := k3d.PrepareGitRepositories(
@@ -560,6 +582,7 @@ func runK3d(cmd *cobra.Command, args []string) error {
 			config.MetaphorDir,
 			&metaphorTemplateTokens,
 			gitProtocolFlag,
+			removeAtlantis,
 		)
 		if err != nil {
 			return err
@@ -992,9 +1015,11 @@ func runK3d(cmd *cobra.Command, args []string) error {
 			log.Error().Err(err).Msg("")
 		}
 
-		err = pkg.OpenBrowser(pkg.ArgoCDLocalURLTLS)
-		if err != nil {
-			log.Error().Err(err).Msg("")
+		if os.Getenv("SKIP_ARGOCD_LAUNCH") != "true" {
+			err = pkg.OpenBrowser(pkg.ArgoCDLocalURLTLS)
+			if err != nil {
+				log.Error().Err(err).Msg("")
+			}
 		}
 	}
 
@@ -1216,6 +1241,7 @@ func runK3d(cmd *cobra.Command, args []string) error {
 		tfEnvs["AWS_SECRET_ACCESS_KEY"] = pkg.MinioDefaultPassword
 		tfEnvs["TF_VAR_aws_access_key_id"] = pkg.MinioDefaultUsername
 		tfEnvs["TF_VAR_aws_secret_access_key"] = pkg.MinioDefaultPassword
+		tfEnvs["TF_VAR_ngrok_authtoken"] = viper.GetString("secrets.atlantis-ngrok-authtoken")
 		// tfEnvs["TF_LOG"] = "DEBUG"
 
 		tfEntrypoint := config.GitopsDir + "/terraform/vault"
@@ -1334,7 +1360,7 @@ func runK3d(cmd *cobra.Command, args []string) error {
 	// Set flags used to track status of active options
 	helpers.SetClusterStatusFlags(k3d.CloudProvider, config.GitProvider)
 
-	cluster := utilities.CreateClusterRecordFromRaw(useTelemetryFlag, cGitOwner, cGitUser, cGitToken, cGitlabOwnerGroupID, gitopsTemplateURLFlag, gitopsTemplateBranchFlag)
+	cluster := utilities.CreateClusterRecordFromRaw(useTelemetryFlag, cGitOwner, cGitUser, cGitToken, cGitlabOwnerGroupID, gitopsTemplateURLFlag, gitopsTemplateBranchFlag, catalogApps)
 
 	err = utilities.ExportCluster(cluster, kcfg)
 	if err != nil {