From 9c566143f57e5850b8d5dd72f6c1d09d3cbae766 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 09:52:02 +0100 Subject: [PATCH 1/8] reordered globe fields added _env remove unused fields --- internal/myks/globe.go | 58 +++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 03fa3d2a..94fc49c6 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -17,60 +17,63 @@ import ( const GlobalLogFormat = "\033[1m[global]\033[0m %s" // Define the main structure +// Globe configuration type Globe struct { - /// Globe configuration - // Directory of application-specific configuration - AppsDir string `default:"_apps"` - // Directory of application-specific prototype overwrites - PrototypeOverrideDir string `default:"_proto"` + // Project root directory + RootDir string `default:"."` // Base directory for environments EnvironmentBaseDir string `default:"envs"` - // Main branch name - MainBranchName string `default:"main"` - // Prefix for kubernetes namespaces - NamespacePrefix string `default:""` // Application prototypes directory PrototypesDir string `default:"prototypes"` + // Ytt library directory name + YttLibraryDirName string `default:"lib"` // Rendered kubernetes manifests directory RenderedDir string `default:"rendered"` - // Project root directory - RootDir string `default:"."` - /// Globe constants + // Directory of application-specific configuration + AppsDir string `default:"_apps"` + // Directory of environment-specific configuration + EnvsDir string `default:"_env"` + // Directory of application-specific prototype overwrites + PrototypeOverrideDir string `default:"_proto"` - // Application data file name - ApplicationDataFileName string `default:"app-data.ytt.yaml"` - // ArgoCD data directory name - ArgoCDDataDirName string `default:"argocd"` // Data values schema file name DataSchemaFileName string `default:"data-schema.ytt.yaml"` + // Application data file name + ApplicationDataFileName string `default:"app-data.ytt.yaml"` // Environment data file name EnvironmentDataFileName string `default:"env-data.ytt.yaml"` - // Helm charts directory name - HelmChartsDirName string `default:"charts"` - // Myks runtime data file name - MyksDataFileName string `default:"myks-data.ytt.yaml"` // Rendered environment data file name RenderedEnvironmentDataFileName string `default:"env-data.yaml"` - // Static files directory name - StaticFilesDirName string `default:"static"` + // Myks runtime data file name + MyksDataFileName string `default:"myks-data.ytt.yaml"` // Service directory name ServiceDirName string `default:".myks"` // Temporary directory name TempDirName string `default:"tmp"` + // Rendered vendir config file name VendirConfigFileName string `default:"vendir.yaml"` // Rendered vendir lock file name VendirLockFileName string `default:"vendir.lock.yaml"` - // Rendered vendir sync file name - VendirSyncFileName string `default:"vendir.sync.yaml"` // Prefix for vendir secret environment variables VendirSecretEnvPrefix string `default:"VENDIR_SECRET_"` + // Downloaded third-party sources VendorDirName string `default:"vendor"` - // Ytt library directory name - YttLibraryDirName string `default:"lib"` + // Helm charts directory name + HelmChartsDirName string `default:"charts"` + + // Plugin subdirectories + // ArgoCD data directory name + ArgoCDDataDirName string `default:"argocd"` + // Helm step directory name + HelmStepDirName string `default:"helm"` + // Static files directory name + StaticFilesDirName string `default:"static"` + // Vendir step directory name + VendirStepDirName string `default:"vendir"` // Ytt step directory name YttPkgStepDirName string `default:"ytt-pkg"` // Ytt step directory name @@ -85,6 +88,9 @@ type Globe struct { // Git repository URL GitRepoUrl string + // Prefix for kubernetes namespacesM only used in helm rendering + NamespacePrefix string `default:""` + // Collected environments for processing environments map[string]*Environment From 2ac457ef27a6a91839912eae22b351e28ea4cfe4 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:00:54 +0100 Subject: [PATCH 2/8] replaced _env everywhere --- internal/myks/plugin_argocd.go | 4 ++-- internal/myks/plugin_static_files.go | 2 +- internal/myks/render_global_ytt.go | 2 +- internal/myks/smart_mode.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index fbb05a73..39d049e6 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -39,7 +39,7 @@ func (e *Environment) renderArgoCD() (err error) { // 1. Collection of environment main data values and schemas yttFiles := e.collectBySubpath(e.g.EnvironmentDataFileName) // 2. Collection of environment argocd-specific data values and schemas, and overlays - yttFiles = append(yttFiles, e.collectBySubpath(filepath.Join("_env", e.g.ArgoCDDataDirName))...) + yttFiles = append(yttFiles, e.collectBySubpath(filepath.Join(e.g.EnvsDir, e.g.ArgoCDDataDirName))...) res, err := e.yttS( "create ArgoCD project yaml", @@ -82,7 +82,7 @@ func (a *Application) renderArgoCD() (err error) { yttFiles = append(yttFiles, prototypeArgoCDDir) } // 4. Collection of environment argocd-specific data values and schemas, and overlays - yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_env", a.e.g.ArgoCDDataDirName))...) + yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join(a.e.g.EnvsDir, a.e.g.ArgoCDDataDirName))...) // 5. Collection of application argocd-specific data values and schemas, and overlays yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ArgoCDDataDirName))...) diff --git a/internal/myks/plugin_static_files.go b/internal/myks/plugin_static_files.go index 470e3a16..169ee6f8 100644 --- a/internal/myks/plugin_static_files.go +++ b/internal/myks/plugin_static_files.go @@ -27,7 +27,7 @@ func (a *Application) copyStaticFiles() (err error) { logStaticFiles(staticFilesDirs) // 3. Static files from the environment - staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_env", a.e.g.StaticFilesDirName))...) + staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join(a.e.g.EnvsDir, a.e.g.StaticFilesDirName))...) logStaticFiles(staticFilesDirs) // 4. Static files from the application staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.StaticFilesDirName))...) diff --git a/internal/myks/render_global_ytt.go b/internal/myks/render_global_ytt.go index 49359fd5..e0775622 100644 --- a/internal/myks/render_global_ytt.go +++ b/internal/myks/render_global_ytt.go @@ -29,7 +29,7 @@ func (g *GlobalYtt) Render(previousStepFile string) (string, error) { yttFiles = append(yttFiles, previousStepFile) } - yttFiles = append(yttFiles, g.app.e.collectBySubpath(filepath.Join("_env", g.app.e.g.YttStepDirName))...) + yttFiles = append(yttFiles, g.app.e.collectBySubpath(filepath.Join(g.app.e.g.EnvsDir, g.app.e.g.YttStepDirName))...) if len(yttFiles) == 0 { log.Debug().Msg(g.app.Msg(globalYttStepName, "No ytt files found")) diff --git a/internal/myks/smart_mode.go b/internal/myks/smart_mode.go index 3e20c63c..859f02fb 100644 --- a/internal/myks/smart_mode.go +++ b/internal/myks/smart_mode.go @@ -72,8 +72,8 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap { }, // Env search path is the only submatch "env": { - e("(" + g.EnvironmentBaseDir + ".*)/_env/" + g.YttStepDirName + "/.*"), - e("(" + g.EnvironmentBaseDir + ".*)/_env/" + g.ArgoCDDataDirName + "/.*"), + e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.YttStepDirName + "/.*"), + e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.ArgoCDDataDirName + "/.*"), e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvironmentDataFileName), }, // Prototype name is the only submatch From 2aaa49aa47be05c37526c60e00e566290cc3ac25 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:01:44 +0100 Subject: [PATCH 3/8] ordered smartmode plugins --- internal/myks/smart_mode.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/myks/smart_mode.go b/internal/myks/smart_mode.go index 859f02fb..328997e4 100644 --- a/internal/myks/smart_mode.go +++ b/internal/myks/smart_mode.go @@ -62,7 +62,13 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap { } // Subdirectories of apps and prototypes are named after plugins - plugins := []string{g.YttStepDirName, "helm", "vendir", g.YttPkgStepDirName, g.ArgoCDDataDirName} + plugins := []string{ + g.ArgoCDDataDirName, + g.HelmStepDirName, + g.VendirStepDirName, + g.YttPkgStepDirName, + g.YttStepDirName, + } pluginsPattern := "(?:" + strings.Join(plugins, "|") + ")" exprMap := map[string][]*regexp.Regexp{ From 42e6e67a985ab9975a0410e07d383f1e4b8d2edd Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:22:25 +0100 Subject: [PATCH 4/8] replaced rendered/argocd and rendered/envs everywhere --- internal/myks/bootstrap.go | 2 +- internal/myks/environment.go | 6 +++--- internal/myks/globe.go | 8 +++++--- internal/myks/plugin_argocd.go | 4 ++-- internal/myks/plugins.go | 2 +- internal/myks/render.go | 2 +- internal/myks/smart_mode_test.go | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/myks/bootstrap.go b/internal/myks/bootstrap.go index 47fb68c5..ac266ebc 100644 --- a/internal/myks/bootstrap.go +++ b/internal/myks/bootstrap.go @@ -84,7 +84,7 @@ func (g *Globe) Bootstrap(force, onlyPrint bool, components []string) error { func (g *Globe) createBaseFileStructure(force bool) error { envDir := filepath.Join(g.RootDir, g.EnvironmentBaseDir) protoDir := filepath.Join(g.RootDir, g.PrototypesDir) - renderedDir := filepath.Join(g.RootDir, g.RenderedDir) + renderedDir := filepath.Join(g.RootDir, g.RenderedEnvsDir) gitignoreFile := filepath.Join(g.RootDir, ".gitignore") myksConfigFile := filepath.Join(g.RootDir, ".myks.yaml") diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 86b013fe..e1b25f08 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -164,11 +164,11 @@ func (e *Environment) Cleanup() error { for _, app := range apps { if _, ok := e.foundApplications[app]; !ok { log.Info().Str("app", app).Msg(e.Msg("Removing app as it is not configured")) - err := os.RemoveAll(filepath.Join(e.g.RootDir, e.g.RenderedDir, "envs", e.Id, app)) + err := os.RemoveAll(filepath.Join(e.g.RootDir, e.g.RenderedEnvsDir, e.Id, app)) if err != nil { return fmt.Errorf("unable to remove dir: %w", err) } - err = os.Remove(filepath.Join(e.g.RootDir, e.g.RenderedDir, "argocd", e.Id, getArgoCDAppFileName(app))) + err = os.Remove(filepath.Join(e.g.RootDir, e.g.RenderedArgoDir, e.Id, getArgoCDAppFileName(app))) if err != nil { return fmt.Errorf("unable to remove file: %w", err) } @@ -181,7 +181,7 @@ func (e *Environment) Cleanup() error { // renderedApplications returns list of applications in rendered dir func (e *Environment) renderedApplications() ([]string, error) { apps := []string{} - dirEnvRendered := filepath.Join(e.g.RootDir, e.g.RenderedDir, "envs", e.Id) + dirEnvRendered := filepath.Join(e.g.RootDir, e.g.RenderedEnvsDir, e.Id) files, err := os.ReadDir(dirEnvRendered) if err != nil { if os.IsNotExist(err) { diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 94fc49c6..ac6e35f9 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -29,7 +29,9 @@ type Globe struct { // Ytt library directory name YttLibraryDirName string `default:"lib"` // Rendered kubernetes manifests directory - RenderedDir string `default:"rendered"` + RenderedEnvsDir string `default:"rendered/envs"` + // Rendered argocd manifests directory + RenderedArgoDir string `default:"rendered/argocd"` // Directory of application-specific configuration AppsDir string `default:"_apps"` @@ -245,8 +247,8 @@ func (g *Globe) Cleanup() error { legalEnvs[env.Id] = true } - for _, dir := range [...]string{"argocd", "envs"} { - dirPath := filepath.Join(g.RootDir, g.RenderedDir, dir) + for _, dir := range [...]string{g.RenderedArgoDir, g.RenderedEnvsDir} { + dirPath := filepath.Join(g.RootDir, dir) files, err := os.ReadDir(dirPath) if err != nil { if os.IsNotExist(err) { diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index 39d049e6..f1548634 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -55,7 +55,7 @@ func (e *Environment) renderArgoCD() (err error) { } func (e *Environment) getArgoCDDestinationDir() string { - return filepath.Join(e.g.RootDir, e.g.RenderedDir, "argocd", e.Id) + return filepath.Join(e.g.RootDir, e.g.RenderedArgoDir, e.Id) } func (a *Application) renderArgoCD() (err error) { @@ -140,7 +140,7 @@ func (a *Application) argoCDPrepareDefaults() (filename string, err error) { } func (a *Application) getArgoCDDestinationDir() string { - return filepath.Join(a.e.g.RootDir, a.e.g.RenderedDir, "argocd", a.e.Id) + return filepath.Join(a.e.g.RootDir, a.e.g.RenderedArgoDir, a.e.Id) } func getArgoCDEnvFileName(envName string) string { diff --git a/internal/myks/plugins.go b/internal/myks/plugins.go index fa5e5d44..0d4f20ce 100644 --- a/internal/myks/plugins.go +++ b/internal/myks/plugins.go @@ -134,7 +134,7 @@ func (p PluginCmd) generateEnv(a *Application) (map[string]string, error) { "MYKS_APP": a.Name, "MYKS_APP_PROTOTYPE": a.Prototype, "MYKS_ENV_DIR": a.e.Dir, - "MYKS_RENDERED_APP_DIR": "rendered/envs/" + a.e.Id + "/" + a.Name, // TODO: provide func and use it everywhere + "MYKS_RENDERED_APP_DIR": a.getDestinationDir(), } result, err := a.ytt(p.Name(), "get data values", a.yttDataFiles, "--data-values-inspect") diff --git a/internal/myks/render.go b/internal/myks/render.go index c0cd6643..b3487b47 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -133,7 +133,7 @@ func (a *Application) storeStepResult(output string, stepName string, stepNumber } func (a *Application) getDestinationDir() string { - return filepath.Join(a.e.g.RootDir, a.e.g.RenderedDir, "envs", a.e.Id, a.Name) + return filepath.Join(a.e.g.RootDir, a.e.g.RenderedEnvsDir, a.e.Id, a.Name) } // Generates a file name for each document using kind and name if available diff --git a/internal/myks/smart_mode_test.go b/internal/myks/smart_mode_test.go index 29ebec37..f5378f1d 100644 --- a/internal/myks/smart_mode_test.go +++ b/internal/myks/smart_mode_test.go @@ -316,7 +316,7 @@ func TestGlobe_runSmartMode(t *testing.T) { for env, apps := range tt.rendered { for _, app := range apps { - dir := filepath.Join(tmpDir, g.RenderedDir, "envs", env, app) + dir := filepath.Join(tmpDir, g.RenderedEnvsDir, env, app) if err := createDirectory(dir); err != nil { t.Errorf("failed to create directory %s", dir) } From 8ef742ec3e1e3f3dfba731126faa4ea61ea7a8b7 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:34:25 +0100 Subject: [PATCH 5/8] replaced _apps everywhere --- internal/myks/plugin_static_files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/plugin_static_files.go b/internal/myks/plugin_static_files.go index 169ee6f8..d6a7c9bc 100644 --- a/internal/myks/plugin_static_files.go +++ b/internal/myks/plugin_static_files.go @@ -30,7 +30,7 @@ func (a *Application) copyStaticFiles() (err error) { staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join(a.e.g.EnvsDir, a.e.g.StaticFilesDirName))...) logStaticFiles(staticFilesDirs) // 4. Static files from the application - staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.StaticFilesDirName))...) + staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.StaticFilesDirName))...) logStaticFiles(staticFilesDirs) staticFilesDestination := filepath.Join(a.getDestinationDir(), a.e.g.StaticFilesDirName) From 998d7b571e75224812ddfebccdf52434429b3808 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:35:45 +0100 Subject: [PATCH 6/8] replaced _proto everywhere --- internal/myks/plugin_static_files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/plugin_static_files.go b/internal/myks/plugin_static_files.go index d6a7c9bc..bf6aca60 100644 --- a/internal/myks/plugin_static_files.go +++ b/internal/myks/plugin_static_files.go @@ -23,7 +23,7 @@ func (a *Application) copyStaticFiles() (err error) { } logStaticFiles(staticFilesDirs) // 2. Static files from prototype overrides - staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_proto", a.prototypeDirName(), a.e.g.StaticFilesDirName))...) + staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join(a.e.g.PrototypeOverrideDir, a.prototypeDirName(), a.e.g.StaticFilesDirName))...) logStaticFiles(staticFilesDirs) // 3. Static files from the environment From 5b66d8750596c898f92a66cee8a4939d0d991c23 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 10:49:08 +0100 Subject: [PATCH 7/8] gofumpt --- internal/myks/globe.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/myks/globe.go b/internal/myks/globe.go index ac6e35f9..4732359a 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -19,7 +19,6 @@ const GlobalLogFormat = "\033[1m[global]\033[0m %s" // Define the main structure // Globe configuration type Globe struct { - // Project root directory RootDir string `default:"."` // Base directory for environments From 4a59fb2a9afbb6cebe5eca15a04d4131660f79b9 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Wed, 27 Dec 2023 14:19:33 +0100 Subject: [PATCH 8/8] Update internal/myks/globe.go Co-authored-by: German Lashevich --- internal/myks/globe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 4732359a..4496e9b9 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -89,7 +89,7 @@ type Globe struct { // Git repository URL GitRepoUrl string - // Prefix for kubernetes namespacesM only used in helm rendering + // Prefix for kubernetes namespaces, only used in helm rendering NamespacePrefix string `default:""` // Collected environments for processing