From cfedbde273ad038e86bcd2abc39dc42f17f32635 Mon Sep 17 00:00:00 2001 From: fritzduchardt Date: Wed, 1 Nov 2023 09:53:11 +0100 Subject: [PATCH 01/13] fix(clean-up): Call env cleanup method when executing render and sync (#125) Closes #100 --- cmd/root.go | 2 +- internal/myks/environment.go | 56 ++++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f77123c1..1d77af7a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/logrusorgru/aurora/v4" + aurora "github.com/logrusorgru/aurora/v4" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 238e7e6f..4590676e 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -106,6 +106,37 @@ func (e *Environment) Render(asyncLevel int) error { return e.Cleanup() } +func (e *Environment) SyncAndRender(asyncLevel int, vendirSecrets string) error { + if err := e.renderArgoCD(); err != nil { + return err + } + err := process(asyncLevel, e.Applications, func(item interface{}) error { + app, ok := item.(*Application) + if !ok { + return fmt.Errorf("Unable to cast item to *Application") + } + if err := app.Sync(vendirSecrets); err != nil { + return err + } + yamlTemplatingTools := []YamlTemplatingTool{ + &Helm{ident: "helm", app: app, additive: true}, + &YttPkg{ident: "ytt-pkg", app: app, additive: true}, + &Ytt{ident: "ytt", app: app, additive: false}, + &GlobalYtt{ident: "global-ytt", app: app, additive: false}, + } + if err := app.RenderAndSlice(yamlTemplatingTools); err != nil { + return err + } + return app.renderArgoCD() + }) + if err != nil { + log.Error().Err(err).Msg(e.Msg("Unable to sync and render applications")) + return err + } + + return e.Cleanup() +} + func (e *Environment) Cleanup() error { apps, err := e.renderedApplications() if err != nil { @@ -171,31 +202,6 @@ func (e *Environment) missingApplications() ([]string, error) { return missingApps, nil } -func (e *Environment) SyncAndRender(asyncLevel int, vendirSecrets string) error { - if err := e.renderArgoCD(); err != nil { - return err - } - return process(asyncLevel, e.Applications, func(item interface{}) error { - app, ok := item.(*Application) - if !ok { - return fmt.Errorf("Unable to cast item to *Application") - } - if err := app.Sync(vendirSecrets); err != nil { - return err - } - yamlTemplatingTools := []YamlTemplatingTool{ - &Helm{ident: "helm", app: app, additive: true}, - &YttPkg{ident: "ytt-pkg", app: app, additive: true}, - &Ytt{ident: "ytt", app: app, additive: false}, - &GlobalYtt{ident: "global-ytt", app: app, additive: false}, - } - if err := app.RenderAndSlice(yamlTemplatingTools); err != nil { - return err - } - return app.renderArgoCD() - }) -} - func (e *Environment) setId() error { yamlBytes, err := os.ReadFile(e.EnvironmentDataFile) if err != nil { From 2d06bc912443707d4a7e6f03902bde5ab5f9d6a4 Mon Sep 17 00:00:00 2001 From: fritzduchardt Date: Wed, 1 Nov 2023 09:53:11 +0100 Subject: [PATCH 02/13] fix(clean-up): Call env cleanup method when executing render and sync (#125) Closes #100 --- internal/myks/environment.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 86b013fe..27c5d8af 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -196,7 +196,6 @@ func (e *Environment) renderedApplications() ([]string, error) { apps = append(apps, dir) } } - log.Debug().Strs("apps", apps).Msg(e.Msg("Found rendered applications")) return apps, nil } From 3b0ab670434d06e5e1770b228edcc81cea8f3873 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Sun, 17 Dec 2023 10:51:40 +0100 Subject: [PATCH 03/13] Introduce helm dependencies build --- internal/myks/application.go | 9 ++++--- internal/myks/assets/data-schema.ytt.yaml | 2 ++ internal/myks/render_helm.go | 30 ++++++++++++----------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 2dfeb9f0..f84f77dd 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -35,10 +35,11 @@ type Application struct { } type HelmConfig struct { - Namespace string `yaml:"namespace"` - KubeVersion string `yaml:"kubeVersion"` - IncludeCRDs bool `yaml:"includeCRDs"` - Capabilities []string `yaml:"capabilities"` + Namespace string `yaml:"namespace"` + KubeVersion string `yaml:"kubeVersion"` + IncludeCRDs bool `yaml:"includeCRDs"` + Capabilities []string `yaml:"capabilities"` + BuildDependencies bool `yaml:"buildDependencies"` } var ( diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index 87e51bed..460aee7a 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -98,6 +98,8 @@ helm: kubeVersion: "" #! If defined, passed as a value of `--namespace` for `helm template`. namespace: "" + #! If true, run "helm dependency build" before rendering. This is required for helm charts with dependencies that are pulled from git repositories. + buildDependencies: false #! Configuration of the step that renders ytt-packages. yttPkg: #! A ytt-package can be rendered as a whole, or can contain multiple sub-packages that should be rendered separately. diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 9f677f54..2be88966 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -1,11 +1,11 @@ package myks import ( + "gopkg.in/yaml.v3" "path/filepath" "strings" "github.com/rs/zerolog/log" - yaml "gopkg.in/yaml.v3" ) type Helm struct { @@ -44,35 +44,37 @@ func (h *Helm) Render(_ string) (string, error) { return "", nil } - helmConfig, err := h.getHelmConfig() - if err != nil { - log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to get helm config")) - return "", err - } - var commonHelmArgs []string // FIXME: move Namespace to a per-chart config - if helmConfig.Namespace == "" { - helmConfig.Namespace = h.app.e.g.NamespacePrefix + h.app.Name + if h.app.HelmConfig.Namespace == "" { + h.app.HelmConfig.Namespace = h.app.e.g.NamespacePrefix + h.app.Name } - commonHelmArgs = append(commonHelmArgs, "--namespace", helmConfig.Namespace) + commonHelmArgs = append(commonHelmArgs, "--namespace", h.app.HelmConfig.Namespace) - if helmConfig.KubeVersion != "" { - commonHelmArgs = append(commonHelmArgs, "--kube-version", helmConfig.KubeVersion) + if h.app.HelmConfig.KubeVersion != "" { + commonHelmArgs = append(commonHelmArgs, "--kube-version", h.app.HelmConfig.KubeVersion) } // FIXME: move IncludeCRDs to a per-chart config - if helmConfig.IncludeCRDs { + if h.app.HelmConfig.IncludeCRDs { commonHelmArgs = append(commonHelmArgs, "--include-crds") } - for _, capa := range helmConfig.Capabilities { + for _, capa := range h.app.HelmConfig.Capabilities { commonHelmArgs = append(commonHelmArgs, "--api-versions", capa) } var outputs []string for _, chartDir := range chartDirs { + + if h.app.HelmConfig.BuildDependencies { + _, err := h.app.runCmd(helmStepName, "helm dependencies build", "helm", nil, []string{"dependencies", "build", chartDir, "--skip-refresh"}) + if err != nil { + return "", err + } + } + chartName := filepath.Base(chartDir) var helmValuesFile string if helmValuesFile, err = h.app.prepareValuesFile("helm", chartName); err != nil { From 1808fc998314fcbc0456c08e17c1a6a368d35cd9 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Thu, 21 Dec 2023 19:38:02 +0100 Subject: [PATCH 04/13] Refactor Argo and Helm config --- internal/myks/application.go | 62 ++++++++++++++++++++++++++++------ internal/myks/plugin_argocd.go | 2 +- internal/myks/render.go | 2 +- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index f84f77dd..2cd97f14 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -12,14 +12,17 @@ import ( ) const ( - renderStepName = "render" - syncStepName = "sync" - globalYttStepName = "global-ytt" - yttStepName = "ytt" - yttPkgStepName = "ytt-pkg" - helmStepName = "helm" - sliceStepName = "slice" - initStepName = "init" + renderStepName = "render" + syncStepName = "sync" + globalYttStepName = "global-ytt" + yttStepName = "ytt" + yttPkgStepName = "ytt-pkg" + helmStepName = "helm" + sliceStepName = "slice" + initStepName = "init" + applyStepName = "apply" + crdPrefix = "customresourcedefinition-" + namespaceResourcePrefix = "namespace-" ) type Application struct { @@ -28,10 +31,11 @@ type Application struct { e *Environment - argoCDEnabled bool includeNamespace bool - yttDataFiles []string yttPkgDirs []string + yttDataFiles []string + ArgoConfig *ArgoConfig + HelmConfig *HelmConfig } type HelmConfig struct { @@ -42,6 +46,19 @@ type HelmConfig struct { BuildDependencies bool `yaml:"buildDependencies"` } +type Destination struct { + Namespace string `yaml:"namespace"` +} + +type App struct { + Destination Destination `yaml:"destination"` +} + +type ArgoConfig struct { + Enabled bool `yaml:"enabled"` + App App `yaml:"app"` +} + var ( ErrNoVendirConfig = errors.New("no vendir config found") ApplicationLogFormat = "\033[1m[%s > %s > %s]\033[0m %s" @@ -101,10 +118,33 @@ func (a *Application) Init() error { if err != nil { return err } - a.argoCDEnabled = applicationData.ArgoCD.Enabled a.includeNamespace = applicationData.Render.IncludeNamespace a.yttPkgDirs = applicationData.YttPkg.Dirs + // Transfer ArgoConfig if enabled + var argoData struct { + ArgoConfig ArgoConfig `yaml:"argocd"` + } + err = yaml.Unmarshal(dataYaml, &argoData) + if err != nil { + log.Warn().Err(err).Msg(a.Msg(helmStepName, "Unable to unmarshal argo config")) + return err + } + if argoData.ArgoConfig.Enabled { + a.ArgoConfig = &argoData.ArgoConfig + } + + // Transfer HelmConfig + var helmConfig struct { + Helm HelmConfig + } + err = yaml.Unmarshal(dataYaml, &helmConfig) + if err != nil { + log.Warn().Err(err).Msg(a.Msg(helmStepName, "Unable to unmarshal helm config")) + return err + } + a.HelmConfig = &helmConfig.Helm + return nil } diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index fbb05a73..e7fdd018 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -59,7 +59,7 @@ func (e *Environment) getArgoCDDestinationDir() string { } func (a *Application) renderArgoCD() (err error) { - if !a.argoCDEnabled { + if a.ArgoConfig != nil { log.Debug().Msg(a.Msg(ArgoCDStepName, "ArgoCD is disabled")) return } diff --git a/internal/myks/render.go b/internal/myks/render.go index c0cd6643..281096e4 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -216,7 +216,7 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st return "", err } - resourceValues, err := a.mergeValuesYaml(renderStepName, a.expandTempPath(valuesFileName)) + resourceValues, err := a.mergeValuesYaml(a.expandTempPath(valuesFileName)) if err != nil { return "", err } From f798f503c622e61c0ba6bf9c1250a015c50c4db1 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Sat, 23 Dec 2023 20:41:08 +0100 Subject: [PATCH 05/13] Add logic to helm build as well as add repositories --- internal/myks/application.go | 19 ++++++++--------- internal/myks/render.go | 2 +- internal/myks/render_helm.go | 40 +++++++++++++++++++++++++++++++++++- internal/myks/util.go | 7 +++++++ internal/myks/util_test.go | 23 +++++++++++++++++++++ 5 files changed, 78 insertions(+), 13 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 2cd97f14..aa8bb640 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -12,17 +12,14 @@ import ( ) const ( - renderStepName = "render" - syncStepName = "sync" - globalYttStepName = "global-ytt" - yttStepName = "ytt" - yttPkgStepName = "ytt-pkg" - helmStepName = "helm" - sliceStepName = "slice" - initStepName = "init" - applyStepName = "apply" - crdPrefix = "customresourcedefinition-" - namespaceResourcePrefix = "namespace-" + renderStepName = "render" + syncStepName = "sync" + globalYttStepName = "global-ytt" + yttStepName = "ytt" + yttPkgStepName = "ytt-pkg" + helmStepName = "helm" + sliceStepName = "slice" + initStepName = "init" ) type Application struct { diff --git a/internal/myks/render.go b/internal/myks/render.go index 281096e4..e26a2ed8 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -216,7 +216,7 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st return "", err } - resourceValues, err := a.mergeValuesYaml(a.expandTempPath(valuesFileName)) + resourceValues, err := a.mergeValuesYaml(initStepName, a.expandTempPath(valuesFileName)) if err != nil { return "", err } diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 2be88966..c1cb9890 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -1,6 +1,7 @@ package myks import ( + "fmt" "gopkg.in/yaml.v3" "path/filepath" "strings" @@ -69,7 +70,7 @@ func (h *Helm) Render(_ string) (string, error) { for _, chartDir := range chartDirs { if h.app.HelmConfig.BuildDependencies { - _, err := h.app.runCmd(helmStepName, "helm dependencies build", "helm", nil, []string{"dependencies", "build", chartDir, "--skip-refresh"}) + err = h.helmBuild(chartDir) if err != nil { return "", err } @@ -113,6 +114,43 @@ func (h *Helm) Render(_ string) (string, error) { return strings.Join(outputs, "---\n"), nil } +func (h *Helm) helmBuild(chartDir string) error { + chartPath := filepath.Join(chartDir, "Chart.yaml") + if exists, _ := isExist(chartDir); exists == false { + return fmt.Errorf("can't locate Chart.yaml at: %s", chartPath) + } + + chart, err := unmarshalYamlToMap(chartPath) + if err != nil { + return fmt.Errorf("failure to unmarshal Chart.yaml at: %s", chartPath) + } + + helmCache := h.app.expandTempPath("helm-cache") + cacheArgs := []string{ + "--repository-cache", filepath.Join(helmCache, "repository"), + "--repository-config", filepath.Join(helmCache, "repositories.yaml"), + } + dependencies := chart["dependencies"].([]interface{}) + for _, dependency := range dependencies { + depMap := dependency.(map[string]interface{}) + repo := depMap["repository"].(string) + if strings.HasPrefix(repo, "http") { + args := []string{"repo", "add", createURLSlug(repo), repo} + _, err := h.app.runCmd(helmStepName, "helm repo add", "helm", nil, append(args, cacheArgs...)) + if err != nil { + return fmt.Errorf("failed to add repository %s in %s ", repo, chartPath) + } + } + } + + buildArgs := []string{"dependencies", "build", chartDir} + _, err = h.app.runCmd(helmStepName, "helm dependencies build", "helm", nil, append(buildArgs, cacheArgs...)) + if err != nil { + return fmt.Errorf("failed to build dependencies for chart %s", chartDir) + } + return nil +} + func (h *Helm) getHelmConfig() (HelmConfig, error) { dataValuesYaml, err := h.app.ytt(helmStepName, "get helm config", h.app.yttDataFiles, "--data-values-inspect") if err != nil { diff --git a/internal/myks/util.go b/internal/myks/util.go index ed9b2991..32b83f06 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -412,3 +412,10 @@ func mapToSlice(env map[string]string) []string { } return envSlice } + +func createURLSlug(url string) string { + url = strings.TrimPrefix(url, "http://") + url = strings.TrimPrefix(url, "https://") + url = url[:strings.LastIndex(url, "/")] + return url +} diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index a8029deb..f86704fc 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -489,3 +489,26 @@ func Test_extract(t *testing.T) { }) } } + +func Test_createURLSlug(t *testing.T) { + type args struct { + input string + } + tests := []struct { + name string + args args + want string + }{ + {"bitnami", args{"https://charts.bitnami.com/bitnami"}, "charts.bitnami.com"}, + {"stable", args{"https://charts.helm.sh/stable"}, "charts.helm.sh"}, + {"grafana", args{"https://grafana.github.io/helm-charts"}, "grafana.github.io"}, + {"nginx", args{"https://helm.nginx.com/stable"}, "helm.nginx.com"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createURLSlug(tt.args.input); got != tt.want { + t.Errorf("createURLSlug() = %v, want %v", got, tt.want) + } + }) + } +} From 508434fce17ecd3d5b516fc028b181103c4569da Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Sat, 23 Dec 2023 20:45:22 +0100 Subject: [PATCH 06/13] Formatting --- cmd/smart-mode.go | 5 +++-- internal/myks/render_helm.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/smart-mode.go b/cmd/smart-mode.go index 0cda14b1..7b629013 100644 --- a/cmd/smart-mode.go +++ b/cmd/smart-mode.go @@ -6,11 +6,12 @@ import ( "os" "strings" - "github.com/logrusorgru/aurora/v4" - "github.com/mykso/myks/internal/myks" + aurora "github.com/logrusorgru/aurora/v4" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/mykso/myks/internal/myks" ) const ( diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index c1cb9890..7d3851ce 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -2,11 +2,11 @@ package myks import ( "fmt" - "gopkg.in/yaml.v3" "path/filepath" "strings" "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v3" ) type Helm struct { From 6d98c85362813d6bdd8985bebcd1c928c51fc020 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Tue, 26 Dec 2023 07:25:20 +0100 Subject: [PATCH 07/13] Improve registry slugs. Build Argo Config back. --- internal/myks/application.go | 9 ++------- internal/myks/plugin_argocd.go | 2 +- internal/myks/render.go | 2 +- internal/myks/util.go | 2 +- internal/myks/util_test.go | 8 ++++---- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index aa8bb640..2dc1cff4 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -28,10 +28,10 @@ type Application struct { e *Environment + argoCDEnabled bool includeNamespace bool yttPkgDirs []string yttDataFiles []string - ArgoConfig *ArgoConfig HelmConfig *HelmConfig } @@ -47,13 +47,8 @@ type Destination struct { Namespace string `yaml:"namespace"` } -type App struct { - Destination Destination `yaml:"destination"` -} - type ArgoConfig struct { Enabled bool `yaml:"enabled"` - App App `yaml:"app"` } var ( @@ -128,7 +123,7 @@ func (a *Application) Init() error { return err } if argoData.ArgoConfig.Enabled { - a.ArgoConfig = &argoData.ArgoConfig + a.argoCDEnabled = true } // Transfer HelmConfig diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index e7fdd018..46e681e2 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -59,7 +59,7 @@ func (e *Environment) getArgoCDDestinationDir() string { } func (a *Application) renderArgoCD() (err error) { - if a.ArgoConfig != nil { + if a.argoCDEnabled == false { log.Debug().Msg(a.Msg(ArgoCDStepName, "ArgoCD is disabled")) return } diff --git a/internal/myks/render.go b/internal/myks/render.go index e26a2ed8..c0cd6643 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -216,7 +216,7 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st return "", err } - resourceValues, err := a.mergeValuesYaml(initStepName, a.expandTempPath(valuesFileName)) + resourceValues, err := a.mergeValuesYaml(renderStepName, a.expandTempPath(valuesFileName)) if err != nil { return "", err } diff --git a/internal/myks/util.go b/internal/myks/util.go index 32b83f06..45f16f65 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -416,6 +416,6 @@ func mapToSlice(env map[string]string) []string { func createURLSlug(url string) string { url = strings.TrimPrefix(url, "http://") url = strings.TrimPrefix(url, "https://") - url = url[:strings.LastIndex(url, "/")] + url = strings.ReplaceAll(url, "/", "-") return url } diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index f86704fc..bb725419 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -499,10 +499,10 @@ func Test_createURLSlug(t *testing.T) { args args want string }{ - {"bitnami", args{"https://charts.bitnami.com/bitnami"}, "charts.bitnami.com"}, - {"stable", args{"https://charts.helm.sh/stable"}, "charts.helm.sh"}, - {"grafana", args{"https://grafana.github.io/helm-charts"}, "grafana.github.io"}, - {"nginx", args{"https://helm.nginx.com/stable"}, "helm.nginx.com"}, + {"bitnami", args{"https://charts.bitnami.com/bitnami"}, "charts.bitnami.com-bitnami"}, + {"stable", args{"https://charts.helm.sh/stable"}, "charts.helm.sh-stable"}, + {"grafana", args{"https://grafana.github.io/helm-charts"}, "grafana.github.io-helm-charts"}, + {"nginx", args{"https://helm.nginx.com/stable"}, "helm.nginx.com-stable"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 9e7f8ae9018723ab5b7161918fa6bc760f8655b6 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Wed, 27 Dec 2023 08:37:19 +0100 Subject: [PATCH 08/13] Remove argocd namespace --- internal/myks/application.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 2dc1cff4..1737d481 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -43,10 +43,6 @@ type HelmConfig struct { BuildDependencies bool `yaml:"buildDependencies"` } -type Destination struct { - Namespace string `yaml:"namespace"` -} - type ArgoConfig struct { Enabled bool `yaml:"enabled"` } From 3cff9c0841feea74e3336d2790244eebd57bfc84 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Wed, 27 Dec 2023 12:02:11 +0100 Subject: [PATCH 09/13] Add support for helm oci registries --- internal/myks/render_helm.go | 2 +- internal/myks/util.go | 1 + internal/myks/util_test.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 7d3851ce..de4dfbd5 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -134,7 +134,7 @@ func (h *Helm) helmBuild(chartDir string) error { for _, dependency := range dependencies { depMap := dependency.(map[string]interface{}) repo := depMap["repository"].(string) - if strings.HasPrefix(repo, "http") { + if strings.HasPrefix(repo, "http") || strings.HasPrefix(repo, "oci://") { args := []string{"repo", "add", createURLSlug(repo), repo} _, err := h.app.runCmd(helmStepName, "helm repo add", "helm", nil, append(args, cacheArgs...)) if err != nil { diff --git a/internal/myks/util.go b/internal/myks/util.go index 92f87577..ca941ac5 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -415,6 +415,7 @@ func mapToSlice(env map[string]string) []string { func createURLSlug(url string) string { url = strings.TrimPrefix(url, "http://") url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "oci://") url = strings.ReplaceAll(url, "/", "-") return url } diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index bb725419..ebba1ec8 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -503,6 +503,7 @@ func Test_createURLSlug(t *testing.T) { {"stable", args{"https://charts.helm.sh/stable"}, "charts.helm.sh-stable"}, {"grafana", args{"https://grafana.github.io/helm-charts"}, "grafana.github.io-helm-charts"}, {"nginx", args{"https://helm.nginx.com/stable"}, "helm.nginx.com-stable"}, + {"oci", args{"oci://registry-1.docker.io/bitnamicharts"}, "registry-1.docker.io-bitnamicharts"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From feb7d8b2b93efb88355e7e99e4f5c50cb9f99584 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Wed, 27 Dec 2023 12:15:24 +0100 Subject: [PATCH 10/13] Build back helmconfig refactoring --- internal/myks/application.go | 13 ------------- internal/myks/plugin_argocd.go | 2 +- internal/myks/render_helm.go | 24 +++++++++++++++--------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 1737d481..0fe4c88f 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -32,7 +32,6 @@ type Application struct { includeNamespace bool yttPkgDirs []string yttDataFiles []string - HelmConfig *HelmConfig } type HelmConfig struct { @@ -121,18 +120,6 @@ func (a *Application) Init() error { if argoData.ArgoConfig.Enabled { a.argoCDEnabled = true } - - // Transfer HelmConfig - var helmConfig struct { - Helm HelmConfig - } - err = yaml.Unmarshal(dataYaml, &helmConfig) - if err != nil { - log.Warn().Err(err).Msg(a.Msg(helmStepName, "Unable to unmarshal helm config")) - return err - } - a.HelmConfig = &helmConfig.Helm - return nil } diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index 46e681e2..fbb05a73 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -59,7 +59,7 @@ func (e *Environment) getArgoCDDestinationDir() string { } func (a *Application) renderArgoCD() (err error) { - if a.argoCDEnabled == false { + if !a.argoCDEnabled { log.Debug().Msg(a.Msg(ArgoCDStepName, "ArgoCD is disabled")) return } diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index de4dfbd5..4b07b6f4 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -45,31 +45,37 @@ func (h *Helm) Render(_ string) (string, error) { return "", nil } + helmConfig, err := h.getHelmConfig() + if err != nil { + log.Warn().Err(err).Msg(h.app.Msg(helmStepName, "Unable to get helm config")) + return "", err + } + var commonHelmArgs []string // FIXME: move Namespace to a per-chart config - if h.app.HelmConfig.Namespace == "" { - h.app.HelmConfig.Namespace = h.app.e.g.NamespacePrefix + h.app.Name + if helmConfig.Namespace == "" { + helmConfig.Namespace = h.app.e.g.NamespacePrefix + h.app.Name } - commonHelmArgs = append(commonHelmArgs, "--namespace", h.app.HelmConfig.Namespace) + commonHelmArgs = append(commonHelmArgs, "--namespace", helmConfig.Namespace) - if h.app.HelmConfig.KubeVersion != "" { - commonHelmArgs = append(commonHelmArgs, "--kube-version", h.app.HelmConfig.KubeVersion) + if helmConfig.KubeVersion != "" { + commonHelmArgs = append(commonHelmArgs, "--kube-version", helmConfig.KubeVersion) } // FIXME: move IncludeCRDs to a per-chart config - if h.app.HelmConfig.IncludeCRDs { + if helmConfig.IncludeCRDs { commonHelmArgs = append(commonHelmArgs, "--include-crds") } - for _, capa := range h.app.HelmConfig.Capabilities { + for _, capa := range helmConfig.Capabilities { commonHelmArgs = append(commonHelmArgs, "--api-versions", capa) } var outputs []string for _, chartDir := range chartDirs { - if h.app.HelmConfig.BuildDependencies { + if helmConfig.BuildDependencies { err = h.helmBuild(chartDir) if err != nil { return "", err @@ -116,7 +122,7 @@ func (h *Helm) Render(_ string) (string, error) { func (h *Helm) helmBuild(chartDir string) error { chartPath := filepath.Join(chartDir, "Chart.yaml") - if exists, _ := isExist(chartDir); exists == false { + if exists, _ := isExist(chartDir); !exists { return fmt.Errorf("can't locate Chart.yaml at: %s", chartPath) } From 560aa10f4632a1dc16381833340680141a22aea0 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Thu, 28 Dec 2023 07:43:07 +0100 Subject: [PATCH 11/13] Improve helm repo update logic --- internal/myks/environment.go | 1 + internal/myks/render_helm.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 27c5d8af..86b013fe 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -196,6 +196,7 @@ func (e *Environment) renderedApplications() ([]string, error) { apps = append(apps, dir) } } + log.Debug().Strs("apps", apps).Msg(e.Msg("Found rendered applications")) return apps, nil } diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 4b07b6f4..1f541e89 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -141,7 +141,7 @@ func (h *Helm) helmBuild(chartDir string) error { depMap := dependency.(map[string]interface{}) repo := depMap["repository"].(string) if strings.HasPrefix(repo, "http") || strings.HasPrefix(repo, "oci://") { - args := []string{"repo", "add", createURLSlug(repo), repo} + args := []string{"repo", "add", createURLSlug(repo), repo, "--force-update"} _, err := h.app.runCmd(helmStepName, "helm repo add", "helm", nil, append(args, cacheArgs...)) if err != nil { return fmt.Errorf("failed to add repository %s in %s ", repo, chartPath) @@ -149,7 +149,7 @@ func (h *Helm) helmBuild(chartDir string) error { } } - buildArgs := []string{"dependencies", "build", chartDir} + buildArgs := []string{"dependencies", "build", chartDir, "--skip-refresh"} _, err = h.app.runCmd(helmStepName, "helm dependencies build", "helm", nil, append(buildArgs, cacheArgs...)) if err != nil { return fmt.Errorf("failed to build dependencies for chart %s", chartDir) From 9ac6ca3370b93d8e4af8ec968e99494911ae41f4 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Thu, 28 Dec 2023 19:54:27 +0100 Subject: [PATCH 12/13] revert ArgoCD-related changes --- internal/myks/application.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 0fe4c88f..f84f77dd 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -30,8 +30,8 @@ type Application struct { argoCDEnabled bool includeNamespace bool - yttPkgDirs []string yttDataFiles []string + yttPkgDirs []string } type HelmConfig struct { @@ -42,10 +42,6 @@ type HelmConfig struct { BuildDependencies bool `yaml:"buildDependencies"` } -type ArgoConfig struct { - Enabled bool `yaml:"enabled"` -} - var ( ErrNoVendirConfig = errors.New("no vendir config found") ApplicationLogFormat = "\033[1m[%s > %s > %s]\033[0m %s" @@ -105,21 +101,10 @@ func (a *Application) Init() error { if err != nil { return err } + a.argoCDEnabled = applicationData.ArgoCD.Enabled a.includeNamespace = applicationData.Render.IncludeNamespace a.yttPkgDirs = applicationData.YttPkg.Dirs - // Transfer ArgoConfig if enabled - var argoData struct { - ArgoConfig ArgoConfig `yaml:"argocd"` - } - err = yaml.Unmarshal(dataYaml, &argoData) - if err != nil { - log.Warn().Err(err).Msg(a.Msg(helmStepName, "Unable to unmarshal argo config")) - return err - } - if argoData.ArgoConfig.Enabled { - a.argoCDEnabled = true - } return nil } From e53d1ac524a7a1bd743dd4f9513c697b79c26da0 Mon Sep 17 00:00:00 2001 From: Fritz Duchardt Date: Fri, 29 Dec 2023 09:40:58 +0100 Subject: [PATCH 13/13] Take oci registries out of "helm repo add". --- internal/myks/render_helm.go | 2 +- internal/myks/util_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/myks/render_helm.go b/internal/myks/render_helm.go index 1f541e89..721f45aa 100644 --- a/internal/myks/render_helm.go +++ b/internal/myks/render_helm.go @@ -140,7 +140,7 @@ func (h *Helm) helmBuild(chartDir string) error { for _, dependency := range dependencies { depMap := dependency.(map[string]interface{}) repo := depMap["repository"].(string) - if strings.HasPrefix(repo, "http") || strings.HasPrefix(repo, "oci://") { + if strings.HasPrefix(repo, "http") { args := []string{"repo", "add", createURLSlug(repo), repo, "--force-update"} _, err := h.app.runCmd(helmStepName, "helm repo add", "helm", nil, append(args, cacheArgs...)) if err != nil { diff --git a/internal/myks/util_test.go b/internal/myks/util_test.go index ebba1ec8..bb725419 100644 --- a/internal/myks/util_test.go +++ b/internal/myks/util_test.go @@ -503,7 +503,6 @@ func Test_createURLSlug(t *testing.T) { {"stable", args{"https://charts.helm.sh/stable"}, "charts.helm.sh-stable"}, {"grafana", args{"https://grafana.github.io/helm-charts"}, "grafana.github.io-helm-charts"}, {"nginx", args{"https://helm.nginx.com/stable"}, "helm.nginx.com-stable"}, - {"oci", args{"oci://registry-1.docker.io/bitnamicharts"}, "registry-1.docker.io-bitnamicharts"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {