From 73233ebd7327e8c0ea8653362006e3db2102023f Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 1 Jan 2024 13:19:42 +0100 Subject: [PATCH] fix(sync): allow local paths in vendir config (#191) --- .../dev/_apps/argocd/.myks/vendir.lock.yaml | 2 +- .../dev/_apps/argocd/.myks/vendir.sync.yaml | 1 - .../mykso/dev/_apps/argocd/.myks/vendir.yaml | 2 +- .../_apps/httpbingo/.myks/vendir.lock.yaml | 2 +- .../_apps/httpbingo/.myks/vendir.sync.yaml | 1 - .../dev/_apps/httpbingo/.myks/vendir.yaml | 2 +- .../helm-installation/.myks/vendir.lock.yaml | 2 +- .../helm-installation/.myks/vendir.sync.yaml | 1 - .../_apps/helm-installation/.myks/vendir.yaml | 4 +- .../helm-render-test/vendir/base.ytt.yaml | 3 +- .../_apps/httpbingo/.myks/vendir.lock.yaml | 2 +- .../_apps/httpbingo/.myks/vendir.sync.yaml | 1 - .../dev/_apps/httpbingo/.myks/vendir.yaml | 2 +- .../_apps/httpbingo/.myks/vendir.lock.yaml | 2 +- .../_apps/httpbingo/.myks/vendir.sync.yaml | 1 - .../prod/_apps/httpbingo/.myks/vendir.yaml | 2 +- go.mod | 2 +- go.sum | 8 +-- internal/myks/sync.go | 57 ++++++++----------- internal/myks/sync_test.go | 5 ++ 20 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.sync.yaml delete mode 100644 examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.sync.yaml delete mode 100644 examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml delete mode 100644 examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.sync.yaml delete mode 100644 examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.sync.yaml diff --git a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.lock.yaml b/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.lock.yaml index ef9dfff0..f395d85b 100644 --- a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.lock.yaml +++ b/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.lock.yaml @@ -7,5 +7,5 @@ directories: tags: - v2.7.3 path: . - path: ytt/argocd + path: envs/mykso/dev/_apps/argocd/vendor/ytt/argocd kind: LockConfig diff --git a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.sync.yaml b/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.sync.yaml deleted file mode 100644 index f611e8d2..00000000 --- a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.sync.yaml +++ /dev/null @@ -1 +0,0 @@ -ytt/argocd: c9846652945082356eb1bde564cd69e4c2162b49b698ffc04a5225d93d532f69 diff --git a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.yaml b/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.yaml index 6933f9b7..d356d6c3 100644 --- a/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.yaml +++ b/examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.yaml @@ -1,7 +1,7 @@ apiVersion: vendir.k14s.io/v1alpha1 kind: Config directories: -- path: ytt/argocd +- path: envs/mykso/dev/_apps/argocd/vendor/ytt/argocd contents: - path: . git: diff --git a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.lock.yaml b/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.lock.yaml index bee78839..bab7561c 100644 --- a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.lock.yaml +++ b/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.lock.yaml @@ -5,5 +5,5 @@ directories: appVersion: v2.2.2 version: 0.1.1 path: . - path: charts/httpbingo + path: envs/mykso/dev/_apps/httpbingo/vendor/charts/httpbingo kind: LockConfig diff --git a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.sync.yaml b/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.sync.yaml deleted file mode 100644 index 33977b5b..00000000 --- a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.sync.yaml +++ /dev/null @@ -1 +0,0 @@ -charts/httpbingo: 999605929cb9dc6c51c2dcd176b8ce3266d870860e41590fae9869380cf26ffe diff --git a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.yaml b/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.yaml index b2d28d91..e7f3e246 100644 --- a/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.yaml +++ b/examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.yaml @@ -1,7 +1,7 @@ apiVersion: vendir.k14s.io/v1alpha1 kind: Config directories: -- path: charts/httpbingo +- path: envs/mykso/dev/_apps/httpbingo/vendor/charts/httpbingo contents: - path: . helmChart: diff --git a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml b/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml index fbb7cb6c..ed3ac742 100644 --- a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml +++ b/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml @@ -3,5 +3,5 @@ directories: - contents: - directory: {} path: . - path: charts/render-test + path: envs/dev/_apps/helm-installation/vendor/charts/render-test kind: LockConfig diff --git a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml b/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml deleted file mode 100644 index c00a1828..00000000 --- a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml +++ /dev/null @@ -1 +0,0 @@ -charts/render-test: f6e95b613adfa1959faef2c0d661b198dcfa7b9058d89dd76418ed58d67b5f95 diff --git a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.yaml b/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.yaml index a266e602..cfaaf128 100644 --- a/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.yaml +++ b/examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.yaml @@ -1,9 +1,9 @@ apiVersion: vendir.k14s.io/v1alpha1 kind: Config directories: -- path: charts/render-test +- path: envs/dev/_apps/helm-installation/vendor/charts/render-test contents: - path: . directory: name: render-test - path: ../../../../../../charts/render-test-chart + path: ../charts/render-test-chart diff --git a/examples/integration-tests/prototypes/helm-render-test/vendir/base.ytt.yaml b/examples/integration-tests/prototypes/helm-render-test/vendir/base.ytt.yaml index 39cfb15c..a773634f 100644 --- a/examples/integration-tests/prototypes/helm-render-test/vendir/base.ytt.yaml +++ b/examples/integration-tests/prototypes/helm-render-test/vendir/base.ytt.yaml @@ -9,5 +9,4 @@ directories: contents: - path: . directory: - name: #@ app.name - path: ../../../../../../charts/render-test-chart + path: ../charts/render-test-chart diff --git a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.lock.yaml b/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.lock.yaml index bee78839..ada7e87a 100644 --- a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.lock.yaml +++ b/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.lock.yaml @@ -5,5 +5,5 @@ directories: appVersion: v2.2.2 version: 0.1.1 path: . - path: charts/httpbingo + path: envs/dev/_apps/httpbingo/vendor/charts/httpbingo kind: LockConfig diff --git a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.sync.yaml b/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.sync.yaml deleted file mode 100644 index 33977b5b..00000000 --- a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.sync.yaml +++ /dev/null @@ -1 +0,0 @@ -charts/httpbingo: 999605929cb9dc6c51c2dcd176b8ce3266d870860e41590fae9869380cf26ffe diff --git a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.yaml b/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.yaml index b2d28d91..965fd1e5 100644 --- a/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.yaml +++ b/examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.yaml @@ -1,7 +1,7 @@ apiVersion: vendir.k14s.io/v1alpha1 kind: Config directories: -- path: charts/httpbingo +- path: envs/dev/_apps/httpbingo/vendor/charts/httpbingo contents: - path: . helmChart: diff --git a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.lock.yaml b/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.lock.yaml index 524e4388..8a0f97db 100644 --- a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.lock.yaml +++ b/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.lock.yaml @@ -5,5 +5,5 @@ directories: appVersion: v2.2.2 version: 0.1.0 path: . - path: charts/httpbingo + path: envs/prod/_apps/httpbingo/vendor/charts/httpbingo kind: LockConfig diff --git a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.sync.yaml b/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.sync.yaml deleted file mode 100644 index 03a730d6..00000000 --- a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.sync.yaml +++ /dev/null @@ -1 +0,0 @@ -charts/httpbingo: 826156b26781f251f788d011da6fd14c10921395e77d1ad4618e316aac07720f diff --git a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.yaml b/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.yaml index 542b7810..3320422a 100644 --- a/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.yaml +++ b/examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.yaml @@ -1,7 +1,7 @@ apiVersion: vendir.k14s.io/v1alpha1 kind: Config directories: -- path: charts/httpbingo +- path: envs/prod/_apps/httpbingo/vendor/charts/httpbingo contents: - path: . helmChart: diff --git a/go.mod b/go.mod index b51ded37..f7abb1b8 100644 --- a/go.mod +++ b/go.mod @@ -109,4 +109,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace carvel.dev/vendir => github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6 // unique-tmp-dir +replace carvel.dev/vendir => github.com/mykso/vendir v0.38.0-unique-tmp-dir diff --git a/go.sum b/go.sum index 5b58c885..e868ca8c 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65 h1:+3J1K6yQFR github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65/go.mod h1:2w+qxVu2KSGW78Ex/XaIqfh/OvBgjEsmN53S4T8vEyA= github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835 h1:mYQweUIBD+TBRjIeQnJmXr0GSVMpI6O0takyb/aaOgo= github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835/go.mod h1:dYeVsKp1vvK8XjdTPR1gF+uk+9doxKeO3hqQTOCr7T4= -github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 h1:MjRdR01xh0sfkeS3OOBv+MYkYsrbHuTDc4rfBnVdFaI= -github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14/go.mod h1:AlgTssDlstr4mf92TR4DPITLfl5+7wEY4cKStCmeeto= github.com/cppforlife/go-cli-ui v0.0.0-20220622150351-995494831c6c h1:Au0iPWQ6E8TMil9HiGW/DI4CBttUpBOZWkzgqwq+PLg= github.com/cppforlife/go-cli-ui v0.0.0-20220622150351-995494831c6c/go.mod h1:ci7nWkU0g40w486NlpUpXXpTD3pkOBjH090Uc0wpER4= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -241,10 +239,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6 h1:P44L6H/jK3ju8K6QJpFHE/5/FIfX6KfZxAk64BOfNkY= -github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8= -github.com/mykso/vendir v0.0.0-20231230003508-84b2be48e389 h1:i4g6aYWEDCpqO2xJ6vi5IfM/cApV9D/A1SJwSbDQ9BY= -github.com/mykso/vendir v0.0.0-20231230003508-84b2be48e389/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8= +github.com/mykso/vendir v0.38.0-unique-tmp-dir h1:xaqkHV4OEOnKgnt22WN6gOnSSENhFdz1BsEQF82M/QI= +github.com/mykso/vendir v0.38.0-unique-tmp-dir/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/internal/myks/sync.go b/internal/myks/sync.go index 4917d7a4..38532b44 100644 --- a/internal/myks/sync.go +++ b/internal/myks/sync.go @@ -3,6 +3,7 @@ package myks import ( _ "embed" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -11,6 +12,15 @@ import ( "github.com/rs/zerolog/log" ) +const vendorDirOverlayTemplate = ` +#@ load("@ytt:overlay", "overlay") +#@overlay/match by=overlay.subset({"kind": "Config", "apiVersion": "vendir.k14s.io/v1alpha1"}) +--- +directories: + - #@overlay/match by=overlay.all, expects="1+" + #@overlay/replace via=lambda l, r: (r + "/" + l).replace("//", "/") + path: %s` + func (a *Application) Sync(vendirSecrets string) error { log.Debug().Msg(a.Msg(syncStepName, "Starting")) if err := a.prepareSync(); err != nil { @@ -49,7 +59,9 @@ func (a *Application) prepareSync() error { return ErrNoVendirConfig } - vendirConfig, err := a.ytt(syncStepName, "creating vendir config", yttFiles) + vendorDir := a.expandPath(a.e.g.VendorDirName) + overlayReader := strings.NewReader(fmt.Sprintf(vendorDirOverlayTemplate, vendorDir)) + vendirConfig, err := a.yttS(syncStepName, "creating vendir config", yttFiles, overlayReader) if err != nil { return err } @@ -76,30 +88,23 @@ func (a *Application) prepareSync() error { } func (a *Application) doSync(vendirSecrets string) error { - // Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`) - vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName) - vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName) - vendorDir := a.expandPath(a.e.g.VendorDirName) - - if err := createDirectory(vendorDir); err != nil { - return err - } + vendirConfigPath := a.expandServicePath(a.e.g.VendirConfigFileName) + vendirLockFilePath := a.expandServicePath(a.e.g.VendirLockFileName) // TODO sync retry - if err := a.runVendirSync(vendorDir, vendirConfigFileRelativePath, vendirLockFileRelativePath, vendirSecrets); err != nil { + if err := a.runVendirSync(vendirConfigPath, vendirLockFilePath, vendirSecrets); err != nil { log.Error().Err(err).Msg(a.Msg(syncStepName, "Vendir sync failed")) return err } - vendirConfigFile := a.expandServicePath(a.e.g.VendirConfigFileName) - return a.cleanupVendorDir(vendorDir, vendirConfigFile) + vendorDir := a.expandPath(a.e.g.VendorDirName) + return a.cleanupVendorDir(vendorDir, vendirConfigPath) } -func (a *Application) runVendirSync(targetDir string, vendirConfig string, vendirLock string, vendirSecrets string) error { +func (a *Application) runVendirSync(vendirConfig, vendirLock, vendirSecrets string) error { args := []string{ "vendir", "sync", - "--chdir=" + targetDir, "--file=" + vendirConfig, "--lock-file=" + vendirLock, "--file=-", @@ -129,7 +134,7 @@ func (a Application) cleanupVendorDir(vendorDir, vendirConfigFile string) error dirs = append(dirs, filepath.Clean(path)+string(filepath.Separator)) } - log.Debug().Strs("managed dirs", dirs).Msg("") + log.Debug().Strs("vendir-managed dirs", dirs).Msg("") return filepath.WalkDir(vendorDir, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -139,32 +144,20 @@ func (a Application) cleanupVendorDir(vendorDir, vendirConfigFile string) error if !d.IsDir() { return nil } - log.Debug().Msg(a.Msg(syncStepName, "Checking directory "+path)) - - relPath, err := filepath.Rel(vendorDir, path) - if err != nil { - return err - } - if relPath == "." { - return nil - } - relPath = relPath + string(filepath.Separator) + path = path + string(filepath.Separator) for _, dir := range dirs { - log.Debug().Str("dir", dir).Str("relPath", relPath).Msg("Checking dir") - if dir == relPath { - log.Debug().Msgf("%s == %s", dir, relPath) + if dir == path { return fs.SkipDir } - if strings.HasPrefix(dir, relPath) { - log.Debug().Msgf("%s has prefix %s", dir, relPath) + if strings.HasPrefix(dir, path) { return nil } // This should never happen - if strings.HasPrefix(relPath, dir) { - log.Debug().Msgf("%s has prefix %s", relPath, dir) + if strings.HasPrefix(path, dir) { + log.Debug().Msgf("%s has prefix %s", path, dir) return fs.SkipDir } } diff --git a/internal/myks/sync_test.go b/internal/myks/sync_test.go index e6ae1be5..66366863 100644 --- a/internal/myks/sync_test.go +++ b/internal/myks/sync_test.go @@ -3,6 +3,7 @@ package myks import ( "os" "path/filepath" + "regexp" "testing" "github.com/creasty/defaults" @@ -131,6 +132,7 @@ func Test_cleanupVendorDir(t *testing.T) { }, } + vendorPathRegex := regexp.MustCompile(`path: (\S*)`) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { const keeperFile = "keeper" @@ -143,6 +145,9 @@ func Test_cleanupVendorDir(t *testing.T) { t.Errorf("creating directory %s error = %v", vendorDir, err) } + // Prepend vendor directory to paths in vendir config + tt.vendirYaml = vendorPathRegex.ReplaceAllString(tt.vendirYaml, "path: "+vendorDir+"/$1") + // Create vendir config file if err := os.MkdirAll(filepath.Dir(vendirConfigFile), 0o755); err != nil { t.Errorf("creating directory %s error = %v", filepath.Dir(vendirConfigFile), err)