diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index fd34f69..df29af8 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -70,7 +70,7 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v3 with: - args: --build-tags integration -p bugs -p unused --timeout=3m + args: --build-tags integration -p bugs -p unused --timeout=10m - name: Test run: | diff --git a/Makefile b/Makefile index a63f895..e73d9b3 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ GO_RUN_ARG := -run $(GO_RUN) endif .PHONY: build -build: +build: generate-examples go mod tidy go build -ldflags "$(LINKMODE)" -tags 'osusergo netgo static_build' -o bin/backup-restore-sidecar github.com/metal-stack/backup-restore-sidecar/cmd strip bin/backup-restore-sidecar @@ -31,10 +31,14 @@ build: test: build go test -cover ./... +.PHONY: generate-examples +generate-examples: + go run ./pkg/generate/examples/dump.go + .PHONY: test-integration test-integration: kind-cluster-create kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest - KUBECONFIG=$(KUBECONFIG) go test $(GO_RUN_ARG) -tags=integration -count 1 -v -p 1 -timeout 10m ./... + KUBECONFIG=$(KUBECONFIG) go test $(GO_RUN_ARG) -tags=integration -count 1 -v -p 1 -timeout 20m ./... .PHONY: proto proto: @@ -60,6 +64,10 @@ start-rethinkdb: start-etcd: $(MAKE) start DB=etcd +.PHONY: start-meilisearch +start-meilisearch: + $(MAKE) start DB=meilisearch + .PHONY: start start: kind-cluster-create kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest diff --git a/README.md b/README.md index 4783438..ad2641f 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,12 @@ Probably, it does not make sense to use this project with large databases. Howev ## Supported Databases -| Database | Image | Status | Upgrade Support | -| --------- | ------------ | :----: | :-------------: | -| postgres | >= 12-alpine | beta | ✅ | -| rethinkdb | >= 2.4.0 | beta | ❌ | -| ETCD | >= 3.5 | alpha | ❌ | +| Database | Image | Status | Upgrade Support | +| ----------- | ------------ | :----: | :-------------: | +| postgres | >= 12-alpine | beta | ✅ | +| rethinkdb | >= 2.4.0 | beta | ❌ | +| ETCD | >= 3.5 | alpha | ❌ | +| meilisearch | >= 1.2.0 | alpha | ✅ | ## Database Upgrades diff --git a/cmd/internal/database/etcd/etcd.go b/cmd/internal/database/etcd/etcd.go index 34f955e..d4e71c8 100644 --- a/cmd/internal/database/etcd/etcd.go +++ b/cmd/internal/database/etcd/etcd.go @@ -43,7 +43,7 @@ func New(log *zap.SugaredLogger, datadir, caCert, cert, key, endpoints, name str } } -// Check checks whether a backup needs to be restored or not, returns true if it needs a backup +// Check indicates whether a restore of the database is required or not. func (db *Etcd) Check(_ context.Context) (bool, error) { empty, err := utils.IsEmpty(db.datadir) if err != nil { diff --git a/cmd/internal/database/meilisearch/meilisearch.go b/cmd/internal/database/meilisearch/meilisearch.go new file mode 100644 index 0000000..05cb812 --- /dev/null +++ b/cmd/internal/database/meilisearch/meilisearch.go @@ -0,0 +1,269 @@ +package meilisearch + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/meilisearch/meilisearch-go" + "github.com/spf13/afero" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" + + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "go.uber.org/zap" +) + +const ( + meilisearchCmd = "meilisearch" + meilisearchVersionFile = "VERSION" + meilisearchDBDir = "data.ms" + latestStableDump = "latest.dump" +) + +// Meilisearch implements the database interface +type Meilisearch struct { + log *zap.SugaredLogger + executor *utils.CmdExecutor + datadir string + copyBinaryAfterBackup bool + + apikey string + client *meilisearch.Client +} + +// New instantiates a new meilisearch database +func New(log *zap.SugaredLogger, datadir string, url string, apikey string) (*Meilisearch, error) { + if url == "" { + return nil, fmt.Errorf("meilisearch api url cannot be empty") + } + if apikey == "" { + return nil, fmt.Errorf("meilisearch api key cannot be empty") + } + + client := meilisearch.NewClient(meilisearch.ClientConfig{ + Host: url, + APIKey: apikey, + }) + + return &Meilisearch{ + log: log, + datadir: datadir, + apikey: apikey, + executor: utils.NewExecutor(log), + client: client, + copyBinaryAfterBackup: true, + }, nil +} + +// Backup takes a dump of meilisearch with the meilisearch client. +func (db *Meilisearch) Backup(ctx context.Context) error { + if err := os.RemoveAll(constants.BackupDir); err != nil { + return fmt.Errorf("could not clean backup directory: %w", err) + } + + if err := os.MkdirAll(constants.BackupDir, 0777); err != nil { + return fmt.Errorf("could not create backup directory: %w", err) + } + + dumpResponse, err := db.client.CreateDump() + if err != nil { + return fmt.Errorf("could not create a dump: %w", err) + } + + db.log.Infow("dump creation triggered", "taskUUID", dumpResponse.TaskUID) + + dumpTask, err := db.client.WaitForTask(dumpResponse.TaskUID, meilisearch.WaitParams{Context: ctx}) + if err != nil { + return err + } + db.log.Infow("dump created successfully", "duration", dumpTask.Duration) + + dumps, err := filepath.Glob(constants.BackupDir + "/*.dump") + if err != nil { + return fmt.Errorf("unable to find dump: %w", err) + } + if len(dumps) != 1 { + return fmt.Errorf("did not find unique dump, found %d", len(dumps)) + } + + // we need to do a copy here and cannot simply rename as the file system is + // mounted by two containers. the dump is created in the database container, + // the copy is done in the backup-restore-sidecar container. os.Rename would + // lead to an error. + + err = utils.Copy(afero.NewOsFs(), dumps[0], path.Join(constants.BackupDir, latestStableDump)) + if err != nil { + return fmt.Errorf("unable to move dump to latest: %w", err) + } + + err = os.Remove(dumps[0]) + if err != nil { + return fmt.Errorf("unable to clean up dump: %w", err) + } + + db.log.Debugw("successfully took backup of meilisearch") + + if db.copyBinaryAfterBackup { + // for a future upgrade, the current meilisearch binary is required + err = db.copyMeilisearchBinary(ctx, true) + if err != nil { + return err + } + } + + return nil +} + +// Check indicates whether a restore of the database is required or not. +func (db *Meilisearch) Check(_ context.Context) (bool, error) { + empty, err := utils.IsEmpty(db.datadir) + if err != nil { + return false, err + } + + if empty { + db.log.Info("data directory is empty") + return true, err + } + + return false, nil +} + +// Probe figures out if the database is running and available for taking backups. +func (db *Meilisearch) Probe(_ context.Context) error { + _, err := db.client.Version() + if err != nil { + return fmt.Errorf("connection error: %w", err) + } + + healthy := db.client.IsHealthy() + if !healthy { + return fmt.Errorf("meilisearch does not report healthiness") + } + + return nil +} + +// Recover restores a database backup +func (db *Meilisearch) Recover(ctx context.Context) error { + dump := path.Join(constants.RestoreDir, latestStableDump) + + if _, err := os.Stat(dump); os.IsNotExist(err) { + return fmt.Errorf("restore file not present: %s", dump) + } + + if err := utils.RemoveContents(db.datadir); err != nil { + return fmt.Errorf("could not clean database data directory: %w", err) + } + + start := time.Now() + + err := db.importDump(ctx, dump) + if err != nil { + return fmt.Errorf("unable to recover %w", err) + } + + db.log.Infow("successfully restored meilisearch database", "duration", time.Since(start).String()) + + return nil +} + +func (db *Meilisearch) importDump(ctx context.Context, dump string) error { + var ( + err error + g, _ = errgroup.WithContext(ctx) + + handleFailedRecovery = func(restoreErr error) error { + db.log.Errorw("trying to handle failed database recovery", "error", restoreErr) + + if err := os.RemoveAll(db.datadir); err != nil { + db.log.Errorw("unable to cleanup database data directory after failed recovery attempt, high risk of starting with fresh database on container restart", "err", err) + } else { + db.log.Info("cleaned up database data directory after failed recovery attempt to prevent start of fresh database") + } + + return restoreErr + } + ) + + args := []string{"--import-dump", dump, "--master-key", db.apikey, "--dump-dir", constants.RestoreDir, "--db-path", db.datadir, "--http-addr", "localhost:1"} + cmd := exec.CommandContext(ctx, meilisearchCmd, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + g.Go(func() error { + db.log.Infow("execute meilisearch", "args", args) + + err = cmd.Run() + if err != nil { + return err + } + + db.log.Info("execution of meilisearch finished without an error") + + return nil + }) + + restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey) + if err != nil { + return fmt.Errorf("unable to create prober") + } + + waitForRestore := func() error { + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + sem := semaphore.NewWeighted(1) + + for { + select { + case <-ticker.C: + if !sem.TryAcquire(1) { + continue + } + + err = restoreDB.Probe(ctx) + sem.Release(1) + if err != nil { + db.log.Errorw("meilisearch is still restoring, continue probing for readiness...", "error", err) + continue + } + + db.log.Infow("meilisearch started after importing the dump, stopping it again for takeover from the database container") + + return nil + case <-ctx.Done(): + return fmt.Errorf("context cancelled during meilisearch restore") + } + } + } + + if err := waitForRestore(); err != nil { + return handleFailedRecovery(err) + } + + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + return handleFailedRecovery(err) + } + + err = g.Wait() + if err != nil { + // will probably work better in meilisearch v1.4.0: https://github.com/meilisearch/meilisearch/commit/eff8570f591fe32a6106087807e3fe8c18e8e5e4 + if strings.Contains(err.Error(), "interrupt") { + db.log.Infow("meilisearch terminated but reported an error which can be ignored", "error", err) + } else { + return handleFailedRecovery(err) + } + } + + db.log.Info("successfully restored meilisearch database") + + return nil +} diff --git a/cmd/internal/database/meilisearch/upgrade.go b/cmd/internal/database/meilisearch/upgrade.go new file mode 100644 index 0000000..5e5aed5 --- /dev/null +++ b/cmd/internal/database/meilisearch/upgrade.go @@ -0,0 +1,236 @@ +package meilisearch + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/avast/retry-go/v4" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "golang.org/x/sync/errgroup" +) + +// Upgrade performs an upgrade of the database in case a newer version of the database is detected. +func (db *Meilisearch) Upgrade(ctx context.Context) error { + start := time.Now() + + versionFile := path.Join(db.datadir, meilisearchVersionFile) + if _, err := os.Stat(versionFile); errors.Is(err, fs.ErrNotExist) { + db.log.Infof("%q is not present, no upgrade required", versionFile) + return nil + } + + dbVersion, err := db.getDatabaseVersion(versionFile) + if err != nil { + return err + } + + meilisearchVersion, err := db.getBinaryVersion(ctx) + if err != nil { + db.log.Errorw("unable to get binary version, skipping upgrade", "error", err) + return nil + } + + if dbVersion.String() == meilisearchVersion.String() { + db.log.Infow("no version difference, no upgrade required", "database-version", dbVersion, "binary-version", meilisearchVersion) + return nil + } + if dbVersion.GreaterThan(meilisearchVersion) { + db.log.Errorw("database is newer than meilisearch binary, aborting", "database-version", dbVersion, "binary-version", meilisearchVersion) + return fmt.Errorf("database is newer than meilisearch binary") + } + + if ok := utils.IsCommandPresent(db.previousBinaryPath()); !ok { + db.log.Infof("%q is not present, please make sure that at least one backup was taken with the old meilisearch version, skipping upgrade", db.previousBinaryPath()) + return nil + } + + db.log.Infow("start upgrade", "from", dbVersion, "to", meilisearchVersion) + + err = db.dumpWithOldBinary(ctx) + if err != nil { + return fmt.Errorf("unable to create dump with old meilisearch binary: %w", err) + } + + oldVersionDataDir := strings.TrimRight(db.datadir, "/") + ".upgrade" + + err = os.Rename(db.datadir, oldVersionDataDir) + if err != nil { + return fmt.Errorf("cannot move old version data dir out of the way, which could have happened due to a failed recovery attempt, consider manual cleanup: %w", err) + } + + dump := path.Join(constants.BackupDir, latestStableDump) + + err = db.importDump(ctx, dump) + if err != nil { + return fmt.Errorf("unable to import dump with new meilisearch binary: %w", err) + } + + err = os.RemoveAll(oldVersionDataDir) + if err != nil { + db.log.Errorw("unable cleanup old version data dir, consider manual cleanup", "error", err) + } + + db.log.Infow("meilisearch upgrade done and new data in place", "duration", time.Since(start)) + + return nil +} + +// copyMeilisearchBinary is needed to save the old meilisearch binary for a later upgrade +func (db *Meilisearch) copyMeilisearchBinary(ctx context.Context, override bool) error { + binPath, err := exec.LookPath(meilisearchCmd) + if err != nil { + return err + } + + if !override { + if _, err := os.Stat(path.Join(binPath, meilisearchCmd)); err == nil { + db.log.Info("meilisearch binary for later upgrade already in place, not copying") + return nil + } + } + + err = os.RemoveAll(db.previousBinaryPath()) + if err != nil { + return fmt.Errorf("unable to remove old meilisearch bin dir: %w", err) + } + + err = os.MkdirAll(path.Dir(db.previousBinaryPath()), 0777) + if err != nil { + return fmt.Errorf("unable to create versioned bin dir in data directory") + } + + db.log.Infow("copying meilisearch binary for later upgrades", "from", binPath, "to", db.previousBinaryPath()) + + copy := exec.CommandContext(ctx, "cp", "-av", binPath, db.previousBinaryPath()) + copy.Stdout = os.Stdout + copy.Stderr = os.Stderr + err = copy.Run() + if err != nil { + return fmt.Errorf("unable to copy meilisearch binary: %w", err) + } + + return nil +} + +// make sure this is still inside the mounted data directory otherwise the upgrade won't work +func (db *Meilisearch) previousBinaryPath() string { + return path.Join(db.datadir, "..", "previous-binary", "meilisearch") +} + +func (db *Meilisearch) getDatabaseVersion(versionFile string) (*semver.Version, error) { + // cat VERSION + // 1.2.0 + versionBytes, err := os.ReadFile(versionFile) + if err != nil { + return nil, fmt.Errorf("unable to read %q: %w", versionFile, err) + } + + v, err := semver.NewVersion(strings.TrimSpace(string(versionBytes))) + if err != nil { + return nil, fmt.Errorf("unable to parse meilisearch binary version in %q: %w", string(versionBytes), err) + } + + return v, nil +} + +func (db *Meilisearch) getBinaryVersion(ctx context.Context) (*semver.Version, error) { + // meilisearch --version + // 1.2.0 + cmd := exec.CommandContext(ctx, meilisearchCmd, "--version") + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("unable to detect meilisearch binary version: %w", err) + } + + _, binaryVersionString, found := strings.Cut(string(out), "meilisearch ") + if !found { + return nil, fmt.Errorf("unable to detect meilisearch binary version in %q", binaryVersionString) + } + + v, err := semver.NewVersion(strings.TrimSpace(binaryVersionString)) + if err != nil { + return nil, fmt.Errorf("unable to parse meilisearch binary version in %q: %w", binaryVersionString, err) + } + + return v, nil +} + +func (db *Meilisearch) dumpWithOldBinary(ctx context.Context) error { + var ( + err error + g, _ = errgroup.WithContext(ctx) + ) + + args := []string{"--master-key", db.apikey, "--dump-dir", constants.BackupDir, "--db-path", db.datadir, "--http-addr", "localhost:1"} + cmd := exec.CommandContext(ctx, db.previousBinaryPath(), args...) // nolint:gosec + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + g.Go(func() error { + db.log.Infow("execute previous meilisearch version", "args", args) + + err = cmd.Run() + if err != nil { + return err + } + + return nil + }) + + restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey) + if err != nil { + return fmt.Errorf("unable to create prober") + } + + restoreDB.copyBinaryAfterBackup = false + + err = retry.Do(func() error { + err = restoreDB.Probe(ctx) + if err != nil { + db.log.Errorw("meilisearch is still starting, continue probing for readiness...", "error", err) + + return err + } + + db.log.Infow("previous meilisearch started and is now ready for backup") + return nil + }, retry.Context(ctx)) + if err != nil { + return err + } + + err = restoreDB.Backup(ctx) + if err != nil { + return fmt.Errorf("unable to create dump from previous meilisearch version") + } + + db.log.Infow("taken dump from previous meilisearch version, stopping it again") + + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + return err + } + + err = g.Wait() + if err != nil { + // will probably work better in meilisearch v1.4.0: https://github.com/meilisearch/meilisearch/commit/eff8570f591fe32a6106087807e3fe8c18e8e5e4 + if strings.Contains(err.Error(), "interrupt") { + db.log.Infow("meilisearch terminated but reported an error which can be ignored", "error", err) + } else { + return err + } + } + + db.log.Info("successfully took dump with previous meilisearch version") + + return nil +} diff --git a/cmd/internal/database/postgres/postgres.go b/cmd/internal/database/postgres/postgres.go index b6eb0ab..383490d 100644 --- a/cmd/internal/database/postgres/postgres.go +++ b/cmd/internal/database/postgres/postgres.go @@ -46,7 +46,7 @@ func New(log *zap.SugaredLogger, datadir string, host string, port int, user str } } -// Check checks whether a backup needs to be restored or not, returns true if it needs a backup +// Check indicates whether a restore of the database is required or not. func (db *Postgres) Check(_ context.Context) (bool, error) { empty, err := utils.IsEmpty(db.datadir) if err != nil { @@ -157,7 +157,7 @@ func (db *Postgres) Probe(ctx context.Context) error { // TODO: use postgres client to connect conn, err := net.DialTimeout("tcp", net.JoinHostPort(db.host, strconv.Itoa(db.port)), connectionTimeout) if err != nil { - return fmt.Errorf("connection error:%w", err) + return fmt.Errorf("connection error: %w", err) } defer conn.Close() return nil diff --git a/cmd/internal/database/postgres/upgrade.go b/cmd/internal/database/postgres/upgrade.go index acea17d..d1c5dc2 100644 --- a/cmd/internal/database/postgres/upgrade.go +++ b/cmd/internal/database/postgres/upgrade.go @@ -16,6 +16,7 @@ import ( "time" "github.com/Masterminds/semver/v3" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" ) const ( @@ -51,7 +52,7 @@ func (db *Postgres) Upgrade(ctx context.Context) error { // Check if required commands are present for _, command := range requiredCommands { - if ok := db.isCommandPresent(command); !ok { + if ok := utils.IsCommandPresent(command); !ok { db.log.Errorf("%q is not present, skipping upgrade", command) return nil } @@ -84,7 +85,7 @@ func (db *Postgres) Upgrade(ctx context.Context) error { // Check if old pg_config are present and match pgVersion oldPostgresConfigCmd := path.Join(oldPostgresBinDir, postgresConfigCmd) - if ok := db.isCommandPresent(oldPostgresConfigCmd); !ok { + if ok := utils.IsCommandPresent(oldPostgresConfigCmd); !ok { db.log.Infof("%q is not present, please make sure that at least one backup was taken with the old postgres version or restart the backup-restore-sidecar container with the old postgres version before running an upgrade, skipping upgrade", oldPostgresConfigCmd) return nil } @@ -244,19 +245,6 @@ func (db *Postgres) getDatabaseVersion(pgVersionFile string) (int, error) { return pgVersion, nil } -func (db *Postgres) isCommandPresent(command string) bool { - p, err := exec.LookPath(command) - if err != nil { - return false - } - - if _, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) { - return false - } - - return true -} - func (db *Postgres) getBinDir(ctx context.Context, pgConfigCmd string) (string, error) { cmd := exec.CommandContext(ctx, pgConfigCmd, "--bindir") out, err := cmd.CombinedOutput() diff --git a/cmd/internal/database/rethinkdb/rethinkdb.go b/cmd/internal/database/rethinkdb/rethinkdb.go index 620ade6..7928bab 100644 --- a/cmd/internal/database/rethinkdb/rethinkdb.go +++ b/cmd/internal/database/rethinkdb/rethinkdb.go @@ -55,7 +55,7 @@ func New(log *zap.SugaredLogger, datadir string, url string, passwordFile string } } -// Check checks whether a backup needs to be restored or not, returns true if it needs a backup +// Check indicates whether a restore of the database is required or not. func (db *RethinkDB) Check(_ context.Context) (bool, error) { empty, err := utils.IsEmpty(db.datadir) if err != nil { diff --git a/cmd/internal/utils/files.go b/cmd/internal/utils/files.go index a2fceef..102680e 100644 --- a/cmd/internal/utils/files.go +++ b/cmd/internal/utils/files.go @@ -3,7 +3,9 @@ package utils import ( "errors" "io" + "io/fs" "os" + "os/exec" "path/filepath" "github.com/spf13/afero" @@ -65,3 +67,16 @@ func Copy(fs afero.Fs, src, dst string) error { return nil } + +func IsCommandPresent(command string) bool { + p, err := exec.LookPath(command) + if err != nil { + return false + } + + if _, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) { + return false + } + + return true +} diff --git a/cmd/main.go b/cmd/main.go index 1d86ae6..a197462 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/etcd" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/meilisearch" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/postgres" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/rethinkdb" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/initializer" @@ -57,6 +58,9 @@ const ( postgresPasswordFlg = "postgres-password" postgresPortFlg = "postgres-port" + meilisearchURLFlg = "meilisearch-url" + meilisearchAPIKeyFlg = "meilisearch-apikey" + rethinkDBPasswordFileFlg = "rethinkdb-passwordfile" rethinkDBURLFlg = "rethinkdb-url" @@ -144,6 +148,7 @@ var startCmd = &cobra.Command{ metrics.Start(logger.Named("metrics")) initializer.New(logger.Named("initializer"), addr, db, bp, comp, metrics, viper.GetString(databaseDatadirFlg)).Start(stop) + if err := probe.Start(stop, logger.Named("probe"), db); err != nil { return err } @@ -265,7 +270,7 @@ func init() { rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd) rootCmd.PersistentFlags().StringP(logLevelFlg, "", "info", "sets the application log level") - rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd]") + rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd|meilisearch]") rootCmd.PersistentFlags().StringP(databaseDatadirFlg, "", "", "the directory where the database stores its data in") err := viper.BindPFlags(rootCmd.PersistentFlags()) @@ -425,6 +430,17 @@ func initDatabase() error { viper.GetString(etcdEndpoints), viper.GetString(etcdName), ) + case "meilisearch": + var err error + db, err = meilisearch.New( + logger.Named("meilisearch"), + datadir, + viper.GetString(meilisearchURLFlg), + viper.GetString(meilisearchAPIKeyFlg), + ) + if err != nil { + return err + } default: return fmt.Errorf("unsupported database type: %s", dbString) } diff --git a/deploy/etcd-local.yaml b/deploy/etcd-local.yaml index d822603..effd7a9 100644 --- a/deploy/etcd-local.yaml +++ b/deploy/etcd-local.yaml @@ -1,4 +1,5 @@ -# DO NOT EDIT! This is auto-generated by the integration tests +# THESE EXAMPLES ARE GENERATED! +# Use them as a template for your deployment, but do not commit manual changes to these files. --- apiVersion: apps/v1 kind: StatefulSet diff --git a/deploy/meilisearch-local.yaml b/deploy/meilisearch-local.yaml new file mode 100644 index 0000000..c4446fa --- /dev/null +++ b/deploy/meilisearch-local.yaml @@ -0,0 +1,194 @@ +# THESE EXAMPLES ARE GENERATED! +# Use them as a template for your deployment, but do not commit manual changes to these files. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + creationTimestamp: null + labels: + app: meilisearch + name: meilisearch +spec: + replicas: 1 + selector: + matchLabels: + app: meilisearch + serviceName: meilisearch + template: + metadata: + creationTimestamp: null + labels: + app: meilisearch + spec: + containers: + - command: + - backup-restore-sidecar + - wait + image: getmeili/meilisearch:v1.3.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: meilisearch + ports: + - containerPort: 7700 + name: http + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + resources: {} + startupProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 10 + volumeMounts: + - mountPath: /backup + name: backup + - mountPath: /data + name: data + - mountPath: /usr/local/bin/backup-restore-sidecar + name: bin-provision + subPath: backup-restore-sidecar + - mountPath: /etc/backup-restore-sidecar + name: backup-restore-sidecar-config + - command: + - backup-restore-sidecar + - start + - --log-level=debug + env: + - name: BACKUP_RESTORE_SIDECAR_MEILISEARCH_APIKEY + valueFrom: + secretKeyRef: + key: MEILISEARCH_APIKEY + name: meilisearch + - name: BACKUP_RESTORE_SIDECAR_MEILISEARCH_URL + valueFrom: + secretKeyRef: + key: MEILISEARCH_URL + name: meilisearch + image: getmeili/meilisearch:v1.3.0 + name: backup-restore-sidecar + ports: + - containerPort: 8000 + name: grpc + resources: {} + volumeMounts: + - mountPath: /backup + name: backup + - mountPath: /data + name: data + - mountPath: /etc/backup-restore-sidecar + name: backup-restore-sidecar-config + - mountPath: /usr/local/bin/backup-restore-sidecar + name: bin-provision + subPath: backup-restore-sidecar + initContainers: + - command: + - cp + - /backup-restore-sidecar + - /bin-provision + image: ghcr.io/metal-stack/backup-restore-sidecar:latest + imagePullPolicy: IfNotPresent + name: backup-restore-sidecar-provider + resources: {} + volumeMounts: + - mountPath: /bin-provision + name: bin-provision + volumes: + - name: data + persistentVolumeClaim: + claimName: data + - name: backup + persistentVolumeClaim: + claimName: backup + - configMap: + name: backup-restore-sidecar-config-meilisearch + name: backup-restore-sidecar-config + - emptyDir: {} + name: bin-provision + updateStrategy: {} + volumeClaimTemplates: + - metadata: + creationTimestamp: null + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - metadata: + creationTimestamp: null + name: backup + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} +status: + availableReplicas: 0 + replicas: 0 +--- +apiVersion: v1 +data: + config.yaml: | + --- + bind-addr: 0.0.0.0 + db: meilisearch + db-data-directory: /data/data.ms/ + backup-provider: local + backup-cron-schedule: "*/1 * * * *" + object-prefix: meilisearch-test + compression-method: targz + post-exec-cmds: + - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files +kind: ConfigMap +metadata: + creationTimestamp: null + name: backup-restore-sidecar-config-meilisearch +--- +apiVersion: v1 +kind: Secret +metadata: + creationTimestamp: null + name: meilisearch +stringData: + MEILISEARCH_APIKEY: test123! + MEILISEARCH_URL: http://localhost:7700 +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: meilisearch + name: meilisearch +spec: + ports: + - name: "7700" + port: 7700 + targetPort: 7700 + - name: metrics + port: 2112 + targetPort: 2112 + selector: + app: meilisearch +status: + loadBalancer: {} diff --git a/deploy/postgres-local.yaml b/deploy/postgres-local.yaml index ffabd86..b7c2a07 100644 --- a/deploy/postgres-local.yaml +++ b/deploy/postgres-local.yaml @@ -1,4 +1,5 @@ -# DO NOT EDIT! This is auto-generated by the integration tests +# THESE EXAMPLES ARE GENERATED! +# Use them as a template for your deployment, but do not commit manual changes to these files. --- apiVersion: apps/v1 kind: StatefulSet diff --git a/deploy/rethinkdb-local.yaml b/deploy/rethinkdb-local.yaml index 994088f..887ccb2 100644 --- a/deploy/rethinkdb-local.yaml +++ b/deploy/rethinkdb-local.yaml @@ -1,4 +1,5 @@ -# DO NOT EDIT! This is auto-generated by the integration tests +# THESE EXAMPLES ARE GENERATED! +# Use them as a template for your deployment, but do not commit manual changes to these files. --- apiVersion: apps/v1 kind: StatefulSet diff --git a/go.mod b/go.mod index 8a8cbde..d55508a 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/metal-stack/backup-restore-sidecar go 1.21 require ( - cloud.google.com/go/storage v1.32.0 + cloud.google.com/go/storage v1.33.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/avast/retry-go/v4 v4.5.0 - github.com/aws/aws-sdk-go v1.45.2 - github.com/docker/docker v24.0.5+incompatible + github.com/aws/aws-sdk-go v1.45.7 + github.com/docker/docker v24.0.6+incompatible github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/lib/pq v1.10.9 + github.com/meilisearch/meilisearch-go v0.25.0 github.com/metal-stack/metal-lib v0.13.3 github.com/metal-stack/v v1.0.3 github.com/mholt/archiver/v3 v3.5.1 @@ -24,31 +25,32 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.uber.org/zap v1.25.0 golang.org/x/sync v0.3.0 - google.golang.org/api v0.138.0 - google.golang.org/grpc v1.57.0 + google.golang.org/api v0.140.0 + google.golang.org/grpc v1.58.0 google.golang.org/protobuf v1.31.0 gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 - sigs.k8s.io/controller-runtime v0.16.1 + sigs.k8s.io/controller-runtime v0.16.2 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go v0.110.8 // indirect cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.2 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/containerd v1.7.5 // indirect + github.com/containerd/containerd v1.7.6 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect @@ -58,21 +60,22 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.5 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect @@ -89,7 +92,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -100,8 +103,8 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc4 // indirect github.com/opencontainers/runc v1.1.9 // indirect - github.com/opentracing/opentracing-go v1.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -115,34 +118,36 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/ulikunitz/xz v0.5.11 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.49.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect go.etcd.io/etcd/api/v3 v3.5.9 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20230911183012-2d3300fd4832 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230911183012-2d3300fd4832 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect gopkg.in/cenkalti/backoff.v2 v2.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) diff --git a/go.sum b/go.sum index 7e8cd43..645cd18 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -43,8 +43,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.32.0 h1:5w6DxEGOnktmJHarxAOUywxVW9lbNWIzlzzUltG/3+o= -cloud.google.com/go/storage v1.32.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= +cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= +cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -58,16 +58,16 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= -github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= +github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= +github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg= github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I= -github.com/aws/aws-sdk-go v1.45.2 h1:hTong9YUklQKqzrGk3WnKABReb5R8GjbG4Y6dEQfjnk= -github.com/aws/aws-sdk-go v1.45.2/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.7 h1:k4QsvWZhm8409TYeRuTV1P6+j3lLKoe+giFA/j3VAps= +github.com/aws/aws-sdk-go v1.45.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -80,7 +80,6 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -90,12 +89,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/containerd v1.7.5 h1:i9T9XpAWMe11BHMN7pu1BZqOGjXaKTPyz2v+KYOZgkY= -github.com/containerd/containerd v1.7.5/go.mod h1:ieJNCSzASw2shSGYLHx8NAE7WsZ/gEigo5fQ78W5Zvw= +github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= +github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -111,8 +106,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= +github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -127,16 +122,14 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= +github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -161,6 +154,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -234,11 +229,11 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -248,7 +243,6 @@ github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qK github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -262,7 +256,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -277,6 +270,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -304,6 +299,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/meilisearch/meilisearch-go v0.25.0 h1:xIp+8YWterHuDvpdYlwQ4Qp7im3JlRHmSKiP0NvjyXs= +github.com/meilisearch/meilisearch-go v0.25.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/metal-stack/metal-lib v0.13.3 h1:BOhwcKHILmBZd2pz2YMOhj8QxzDaz3G0F/CGuYhnu8o= github.com/metal-stack/metal-lib v0.13.3/go.mod h1:BAR7fjdoV7DDg8i9GpJQBDaNSFirOcBs0vLYTBnhHQU= github.com/metal-stack/v v1.0.3 h1:Sh2oBlnxrCUD+mVpzfC8HiqL045YWkxs0gpTvkjppqs= @@ -312,8 +309,8 @@ github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Cl github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -348,10 +345,11 @@ github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYB github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= -github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= -github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -375,7 +373,6 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -410,6 +407,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -420,6 +418,12 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/fasthttp v1.49.0 h1:9FdvCpmxB74LH4dPb7IJ1cOSsluR07XG3I1txXWwJpE= +github.com/valyala/fasthttp v1.49.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -441,7 +445,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -460,10 +463,10 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -474,8 +477,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -536,10 +539,11 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -549,8 +553,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -604,6 +608,8 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -684,8 +690,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -713,16 +719,17 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.138.0 h1:K/tVp05MxNVbHShRw9m7e9VJGdagNeTdMzqPH7AUqr0= -google.golang.org/api v0.138.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= +google.golang.org/api v0.140.0 h1:CaXNdYOH5oQQI7l6iKTHHiMTdxZca4/02hRg2U8c2hM= +google.golang.org/api v0.140.0/go.mod h1:aGbCiFgtwb2P6badchFbSBUurV6oR5d50Af4iNJtDdI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -747,7 +754,6 @@ google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -761,12 +767,12 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 h1:Iveh6tGCJkHAjJgEqUQYGDGgbwmhjoAOz8kO/ajxefY= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 h1:WGq4lvB/mlicysM/dUT3SBvijH4D3sm/Ny1A4wmt2CI= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20230911183012-2d3300fd4832 h1:/30npZKtUjXqju7ZA2MsvpkGKD4mQFtf+zPnZasABjg= +google.golang.org/genproto v0.0.0-20230911183012-2d3300fd4832/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230911183012-2d3300fd4832 h1:4E7rZzBdR5LmiZx6n47Dg4AjH8JLhMQWywsYqvXNLcs= +google.golang.org/genproto/googleapis/api v0.0.0-20230911183012-2d3300fd4832/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 h1:o4LtQxebKIJ4vkzyhtD2rfUNZ20Zf0ik5YVP5E7G7VE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -780,14 +786,11 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= +google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -821,7 +824,6 @@ gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 h1:tczPZjdz6soV2thcuq1IFOuNLrBUGonFyUX gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2/go.mod h1:c7Wo0IjB7JL9B9Avv0UZKorYJCUhiergpj3u1WtGT1E= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -849,18 +851,18 @@ k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f h1:eeEUOoGYWhOz7EyXqhlR2zHKNw2mNJ9vzJmub6YN6kk= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= -sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/integration/etcd_test.go b/integration/etcd_test.go index b51c7b9..9217929 100644 --- a/integration/etcd_test.go +++ b/integration/etcd_test.go @@ -8,276 +8,25 @@ import ( "time" "github.com/avast/retry-go/v4" - "github.com/metal-stack/backup-restore-sidecar/pkg/constants" - "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" - _ "github.com/lib/pq" -) + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" -var ( - etcdContainerImage = "quay.io/coreos/etcd:v3.5.7" + _ "github.com/lib/pq" ) func Test_ETCD_Restore(t *testing.T) { restoreFlow(t, &flowSpec{ - databaseType: "etcd", - sts: etcdSts, - backingResources: etcdBackingResources, + databaseType: examples.Etcd, + sts: examples.EtcdSts, + backingResources: examples.EtcdBackingResources, addTestData: addEtcdTestData, verifyTestData: verifyEtcdTestData, }) } -func etcdSts(namespace string, image string) *appsv1.StatefulSet { - if image == "" { - image = etcdContainerImage - } - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "StatefulSet", - APIVersion: appsv1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "etcd", - Namespace: namespace, - Labels: map[string]string{ - "app": "etcd", - }, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: "etcd", - Replicas: pointer.Pointer(int32(1)), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "etcd", - }, - }, - Spec: corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{ - { - Name: "etcd", - Image: image, - Command: []string{"backup-restore-sidecar", "wait"}, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"/usr/local/bin/etcdctl", "endpoint", "health", "--endpoints=127.0.0.1:32379"}, - }, - }, - InitialDelaySeconds: 15, - TimeoutSeconds: 1, - PeriodSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 3, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/health", - Port: intstr.FromInt(32381), - Scheme: corev1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 15, - TimeoutSeconds: 1, - PeriodSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 3, - }, - Ports: []corev1.ContainerPort{ - // default ports are taken by kind etcd because running in host network - { - ContainerPort: 32379, - Name: "client", - Protocol: corev1.ProtocolTCP, - }, - { - ContainerPort: 32380, - Name: "server", - Protocol: corev1.ProtocolTCP, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/data", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - }, - }, - { - Name: "backup-restore-sidecar", - Image: image, - Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 8000, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "backup", - MountPath: constants.SidecarBaseDir, - }, - { - Name: "data", - MountPath: "/data", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - }, - }, - }, - InitContainers: []corev1.Container{ - { - Name: "backup-restore-sidecar-provider", - Image: backupRestoreSidecarContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "cp", - "/backup-restore-sidecar", - "/bin-provision", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "bin-provision", - MountPath: "/bin-provision", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "data", - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "backup", - }, - }, - }, - { - Name: "backup-restore-sidecar-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "backup-restore-sidecar-config-postgres", - }, - }, - }, - }, - { - Name: "bin-provision", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "data", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - } -} - -func etcdBackingResources(namespace string) []client.Object { - return []client.Object{ - &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup-restore-sidecar-config-postgres", - Namespace: namespace, - }, - Data: map[string]string{ - "config.yaml": `--- -bind-addr: 0.0.0.0 -db: etcd -db-data-directory: /data/etcd/ -backup-provider: local -backup-cron-schedule: "*/1 * * * *" -object-prefix: etcd-test -etcd-endpoints: http://localhost:32379 -post-exec-cmds: -- etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 -`, - }, - }, - } -} - func newEtcdClient(t *testing.T, ctx context.Context) *clientv3.Client { var cli *clientv3.Client diff --git a/integration/main_test.go b/integration/main_test.go index 7b5fdef..4aaf410 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -7,8 +7,6 @@ import ( "errors" "fmt" "os" - "path" - "runtime" "strings" "testing" "time" @@ -17,11 +15,9 @@ import ( v1 "github.com/metal-stack/backup-restore-sidecar/api/v1" brsclient "github.com/metal-stack/backup-restore-sidecar/pkg/client" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/yaml" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -32,17 +28,19 @@ import ( type flowSpec struct { databaseType string - // slice of images, executed in order during upgrade - databaseImages []string - sts func(namespace, image string) *appsv1.StatefulSet + + sts func(namespace string) *appsv1.StatefulSet backingResources func(namespace string) []client.Object addTestData func(t *testing.T, ctx context.Context) verifyTestData func(t *testing.T, ctx context.Context) } -const ( - backupRestoreSidecarContainerImage = "ghcr.io/metal-stack/backup-restore-sidecar:latest" -) +type upgradeFlowSpec struct { + flowSpec + + // slice of images, executed in order during upgrade + databaseImages []string +} var ( restConfig *rest.Config @@ -65,11 +63,7 @@ func restoreFlow(t *testing.T, spec *flowSpec) { var ( ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) ns = testNamespace(t) - image string ) - if len(spec.databaseImages) > 0 { - image = spec.databaseImages[0] - } defer cancel() @@ -95,22 +89,20 @@ func restoreFlow(t *testing.T, spec *flowSpec) { t.Log("applying resource manifests") objects := func() []client.Object { - objects := []client.Object{spec.sts(ns.Name, image)} + objects := []client.Object{spec.sts(ns.Name)} objects = append(objects, spec.backingResources(ns.Name)...) return objects } - dumpToExamples(t, spec.databaseType+"-local.yaml", objects()...) - for _, o := range objects() { o := o err = c.Create(ctx, o) require.NoError(t, err) } - podName := spec.sts(ns.Name, image).Name + "-0" + podName := spec.sts(ns.Name).Name + "-0" - err = waitForPodRunnig(ctx, podName, ns.Name) + err = waitForPodRunning(ctx, podName, ns.Name) require.NoError(t, err) t.Log("adding test data to database") @@ -147,7 +139,7 @@ func restoreFlow(t *testing.T, spec *flowSpec) { t.Log("remove sts and delete data volume") - err = c.Delete(ctx, spec.sts(ns.Name, image)) + err = c.Delete(ctx, spec.sts(ns.Name)) require.NoError(t, err) err = c.Delete(ctx, &corev1.PersistentVolumeClaim{ @@ -168,10 +160,10 @@ func restoreFlow(t *testing.T, spec *flowSpec) { t.Log("recreate sts") - err = c.Create(ctx, spec.sts(ns.Name, image)) + err = c.Create(ctx, spec.sts(ns.Name)) require.NoError(t, err) - err = waitForPodRunnig(ctx, podName, ns.Name) + err = waitForPodRunning(ctx, podName, ns.Name) require.NoError(t, err) t.Log("verify that data gets restored") @@ -179,7 +171,7 @@ func restoreFlow(t *testing.T, spec *flowSpec) { spec.verifyTestData(t, ctx) } -func upgradeFlow(t *testing.T, spec *flowSpec) { +func upgradeFlow(t *testing.T, spec *upgradeFlowSpec) { t.Log("running upgrade flow") require.GreaterOrEqual(t, len(spec.databaseImages), 2, "at least 2 database images must be specified for the upgrade test") @@ -212,10 +204,15 @@ func upgradeFlow(t *testing.T, spec *flowSpec) { err := c.Create(ctx, ns) require.NoError(t, client.IgnoreAlreadyExists(err)) - t.Log("applying resource manifests") + t.Logf("starting database with initial image %q", initialImage) + + sts := spec.sts(ns.Name) + for i := range sts.Spec.Template.Spec.Containers { + sts.Spec.Template.Spec.Containers[i].Image = initialImage + } objects := func() []client.Object { - objects := []client.Object{spec.sts(ns.Name, initialImage)} + objects := []client.Object{sts} objects = append(objects, spec.backingResources(ns.Name)...) return objects } @@ -226,53 +223,60 @@ func upgradeFlow(t *testing.T, spec *flowSpec) { require.NoError(t, err) } - podName := spec.sts(ns.Name, initialImage).Name + "-0" + podName := spec.sts(ns.Name).Name + "-0" - err = waitForPodRunnig(ctx, podName, ns.Name) + err = waitForPodRunning(ctx, podName, ns.Name) require.NoError(t, err) t.Log("adding test data to database") spec.addTestData(t, ctx) - t.Log("taking a backup") + for _, image := range nextImages { + t.Log("taking a backup") - brsc, err := brsclient.New(ctx, "http://localhost:8000") - require.NoError(t, err) + brsc, err := brsclient.New(ctx, "http://localhost:8000") + require.NoError(t, err) - _, err = brsc.DatabaseServiceClient().CreateBackup(ctx, &v1.CreateBackupRequest{}) - assert.NoError(t, err) + var backup *v1.Backup + err = retry.Do(func() error { + _, err = brsc.DatabaseServiceClient().CreateBackup(ctx, &v1.CreateBackupRequest{}) + if err != nil { + t.Log(err) + return err + } - var backup *v1.Backup - err = retry.Do(func() error { - backups, err := brsc.BackupServiceClient().ListBackups(ctx, &v1.ListBackupsRequest{}) - if err != nil { - return err - } + backups, err := brsc.BackupServiceClient().ListBackups(ctx, &v1.ListBackupsRequest{}) + if err != nil { + return err + } - if len(backups.Backups) == 0 { - return fmt.Errorf("no backups were made yet") - } + if len(backups.Backups) == 0 { + return fmt.Errorf("no backups were made yet") + } - backup = backups.Backups[0] + backup = backups.Backups[0] - return nil - }, retry.Context(ctx), retry.Attempts(0), retry.MaxDelay(2*time.Second)) - require.NoError(t, err) - require.NotNil(t, backup) + return nil + }, retry.Context(ctx), retry.Attempts(0), retry.MaxDelay(2*time.Second)) + require.NoError(t, err) + require.NotNil(t, backup) - for _, image := range nextImages { - image := image - nextSts := spec.sts(ns.Name, image).DeepCopy() + nextSts := spec.sts(ns.Name) + for i := range nextSts.Spec.Template.Spec.Containers { + nextSts.Spec.Template.Spec.Containers[i].Image = image + } t.Logf("deploy sts with next database version %q, container %q", image, nextSts.Spec.Template.Spec.Containers[0].Image) err = c.Update(ctx, nextSts, &client.UpdateOptions{}) require.NoError(t, err) - time.Sleep(10 * time.Second) + time.Sleep(1 * time.Second) + + err = waitForStsRunning(ctx, sts.Name, ns.Name) + require.NoError(t, err) - // TODO maybe better wait for generation changed - err = waitForPodRunnig(ctx, podName, ns.Name) + err = waitForPodRunning(ctx, podName, ns.Name) require.NoError(t, err) t.Log("verify that data is still the same") @@ -321,41 +325,7 @@ func newKubernetesClient() (client.Client, error) { return c, nil } -func dumpToExamples(t *testing.T, name string, resources ...client.Object) { - content := []byte(`# DO NOT EDIT! This is auto-generated by the integration tests ---- -`) - - for i, r := range resources { - r.SetNamespace("") // not needed for example manifests - - r := r.DeepCopyObject() - - if sts, ok := r.(*appsv1.StatefulSet); ok { - // host network is only for integration testing purposes - sts.Spec.Template.Spec.HostNetwork = false - } - - raw, err := yaml.Marshal(r) - require.NoError(t, err) - - if i != len(resources)-1 { - raw = append(raw, []byte("---\n")...) - } - - content = append(content, raw...) - } - - _, filename, _, _ := runtime.Caller(1) - - dest := path.Join(path.Dir(filename), "..", "deploy", name) - t.Logf("example manifest written to %s", dest) - - err := os.WriteFile(dest, content, 0600) - require.NoError(t, err) -} - -func waitForPodRunnig(ctx context.Context, name, namespace string) error { +func waitForPodRunning(ctx context.Context, name, namespace string) error { return retry.Do(func() error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -384,7 +354,33 @@ func waitForPodRunnig(ctx context.Context, name, namespace string) error { } return nil - }, retry.Context(ctx), retry.Attempts(0)) + }, retry.Context(ctx), retry.Attempts(0), retry.MaxDelay(2*time.Second)) +} + +func waitForStsRunning(ctx context.Context, name, namespace string) error { + return retry.Do(func() error { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + err := c.Get(ctx, client.ObjectKeyFromObject(sts), sts) + if err != nil { + return err + } + + if sts.Status.CurrentRevision != sts.Status.UpdateRevision { + return fmt.Errorf("sts revision is not yet running running") + } + + if sts.Status.ReadyReplicas != sts.Status.Replicas { + return fmt.Errorf("not all replicas in ready status") + } + + return nil + }, retry.Context(ctx), retry.Attempts(0), retry.MaxDelay(2*time.Second)) } func waitUntilNotFound(ctx context.Context, obj client.Object) error { @@ -398,5 +394,5 @@ func waitUntilNotFound(ctx context.Context, obj client.Object) error { } return fmt.Errorf("resource is still running: %s", obj.GetName()) - }, retry.Context(ctx), retry.Attempts(0)) + }, retry.Context(ctx), retry.Attempts(0), retry.MaxDelay(2*time.Second)) } diff --git a/integration/meilisearch_test.go b/integration/meilisearch_test.go new file mode 100644 index 0000000..2ccf41e --- /dev/null +++ b/integration/meilisearch_test.go @@ -0,0 +1,99 @@ +//go:build integration + +package integration_test + +import ( + "context" + "fmt" + "testing" + + "github.com/avast/retry-go/v4" + "github.com/meilisearch/meilisearch-go" + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + _ "github.com/lib/pq" +) + +const ( + meilisearchIndex = "backup-restore-sidecar" +) + +func Test_Meilisearch_Restore(t *testing.T) { + restoreFlow(t, &flowSpec{ + databaseType: examples.Meilisearch, + sts: examples.MeilisearchSts, + backingResources: examples.MeilisearchBackingResources, + addTestData: addMeilisearchTestData, + verifyTestData: verifyMeilisearchTestData, + }) +} + +func Test_Meilisearch_Upgrade(t *testing.T) { + upgradeFlow(t, &upgradeFlowSpec{ + flowSpec: flowSpec{ + databaseType: examples.Meilisearch, + sts: examples.MeilisearchSts, + backingResources: examples.MeilisearchBackingResources, + addTestData: addMeilisearchTestData, + verifyTestData: verifyMeilisearchTestData, + }, + databaseImages: []string{ + "getmeili/meilisearch:v1.2.0", + "getmeili/meilisearch:v1.3.0", + "getmeili/meilisearch:v1.3.2", + }, + }) +} + +func newMeilisearchSession(t *testing.T, ctx context.Context) *meilisearch.Client { + var client *meilisearch.Client + err := retry.Do(func() error { + + client = meilisearch.NewClient(meilisearch.ClientConfig{ + Host: "http://localhost:7700", + APIKey: examples.MeilisearchPassword, + }) + + ok := client.IsHealthy() + if !ok { + return fmt.Errorf("meilisearch is not yet healthy") + } + return nil + }, retry.Context(ctx)) + require.NoError(t, err) + + return client +} + +func addMeilisearchTestData(t *testing.T, ctx context.Context) { + client := newMeilisearchSession(t, ctx) + creationTask, err := client.CreateIndex(&meilisearch.IndexConfig{ + Uid: meilisearchIndex, + PrimaryKey: "id", + }) + require.NoError(t, err) + _, err = client.WaitForTask(creationTask.TaskUID) + require.NoError(t, err) + + index := client.Index(meilisearchIndex) + testdata := map[string]any{ + "id": "1", + "key": "I am precious", + } + indexTask, err := index.AddDocuments(testdata, "id") + require.NoError(t, err) + _, err = client.WaitForTask(indexTask.TaskUID) + require.NoError(t, err) +} + +func verifyMeilisearchTestData(t *testing.T, ctx context.Context) { + client := newMeilisearchSession(t, ctx) + index, err := client.GetIndex(meilisearchIndex) + require.NoError(t, err) + testdata := make(map[string]any) + err = index.GetDocument("1", &meilisearch.DocumentQuery{}, &testdata) + require.NoError(t, err) + assert.Equal(t, "I am precious", testdata["key"]) +} diff --git a/integration/postgres_test.go b/integration/postgres_test.go index dd8ffe9..bbdc406 100644 --- a/integration/postgres_test.go +++ b/integration/postgres_test.go @@ -8,407 +8,47 @@ import ( "fmt" "testing" + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" + "github.com/avast/retry-go/v4" - "github.com/metal-stack/backup-restore-sidecar/pkg/constants" - "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" _ "github.com/lib/pq" ) -const ( - postgresDB = "postgres" - postgresPassword = "test123!" - postgresUser = "postgres" -) - -var ( - postgresContainerImage = "postgres:12-alpine" -) - func Test_Postgres_Restore(t *testing.T) { restoreFlow(t, &flowSpec{ - databaseType: "postgres", - sts: postgresSts, - backingResources: postgresBackingResources, + databaseType: examples.Postgres, + sts: examples.PostgresSts, + backingResources: examples.PostgresBackingResources, addTestData: addPostgresTestData, verifyTestData: verifyPostgresTestData, }) } func Test_Postgres_Upgrade(t *testing.T) { - upgradeFlow(t, &flowSpec{ - databaseType: "postgres", + upgradeFlow(t, &upgradeFlowSpec{ + flowSpec: flowSpec{ + databaseType: examples.Postgres, + sts: examples.PostgresSts, + backingResources: examples.PostgresBackingResources, + addTestData: addPostgresTestData, + verifyTestData: verifyPostgresTestData, + }, databaseImages: []string{ "postgres:12-alpine", // "postgres:13-alpine", commented to test if two versions upgrade also work "postgres:14-alpine", "postgres:15-alpine", }, - sts: postgresSts, - backingResources: postgresBackingResources, - addTestData: addPostgresTestData, - verifyTestData: verifyPostgresTestData, }) } -func postgresSts(namespace, image string) *appsv1.StatefulSet { - if image == "" { - image = postgresContainerImage - } - - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "StatefulSet", - APIVersion: appsv1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres", - Namespace: namespace, - Labels: map[string]string{ - "app": "postgres", - }, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: "postgres", - Replicas: pointer.Pointer(int32(1)), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "postgres", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "postgres", - }, - }, - Spec: corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{ - { - Name: "postgres", - Image: image, - Command: []string{"backup-restore-sidecar", "wait"}, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"/bin/sh", "-c", "exec", "pg_isready", "-U", postgresUser, "-h", "127.0.0.1", "-p", "5432"}, - }, - }, - InitialDelaySeconds: 30, - TimeoutSeconds: 5, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 6, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"/bin/sh", "-c", "exec", "pg_isready", "-U", postgresUser, "-h", "127.0.0.1", "-p", "5432"}, - }, - }, - InitialDelaySeconds: 5, - TimeoutSeconds: 5, - PeriodSeconds: 10, - }, - Env: []corev1.EnvVar{ - { - Name: "POSTGRES_DB", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_DB", - }, - }, - }, - { - Name: "POSTGRES_USER", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_USER", - }, - }, - }, - { - Name: "POSTGRES_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_PASSWORD", - }, - }, - }, - { - Name: "PGDATA", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_DATA", - }, - }, - }, - }, - Ports: []corev1.ContainerPort{ - { - ContainerPort: 5432, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/data", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - }, - }, - { - Name: "backup-restore-sidecar", - Image: image, - Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, - Env: []corev1.EnvVar{ - { - Name: "BACKUP_RESTORE_SIDECAR_POSTGRES_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_PASSWORD", - }, - }, - }, - { - Name: "BACKUP_RESTORE_SIDECAR_POSTGRES_USER", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "postgres", - }, - Key: "POSTGRES_USER", - }, - }, - }, - }, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 8000, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "backup", - MountPath: constants.SidecarBaseDir, - }, - { - Name: "data", - MountPath: "/data", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - }, - }, - }, - InitContainers: []corev1.Container{ - { - Name: "backup-restore-sidecar-provider", - Image: backupRestoreSidecarContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "cp", - "/backup-restore-sidecar", - "/bin-provision", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "bin-provision", - MountPath: "/bin-provision", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "data", - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "backup", - }, - }, - }, - { - Name: "backup-restore-sidecar-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "backup-restore-sidecar-config-postgres", - }, - }, - }, - }, - { - Name: "bin-provision", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "data", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - } -} - -func postgresBackingResources(namespace string) []client.Object { - return []client.Object{ - &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup-restore-sidecar-config-postgres", - Namespace: namespace, - }, - Data: map[string]string{ - "config.yaml": `--- -bind-addr: 0.0.0.0 -db: postgres -db-data-directory: /data/postgres/ -backup-provider: local -backup-cron-schedule: "*/1 * * * *" -object-prefix: postgres-test -compression-method: tar -post-exec-cmds: -- docker-entrypoint.sh postgres -`, - }, - }, - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres", - Namespace: namespace, - }, - StringData: map[string]string{ - "POSTGRES_DB": postgresDB, - "POSTGRES_USER": postgresUser, - "POSTGRES_PASSWORD": postgresPassword, - "POSTGRES_DATA": "/data/postgres/", - }, - }, - &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres", - Namespace: namespace, - Labels: map[string]string{ - "app": "postgres", - }, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "postgres", - }, - Ports: []corev1.ServicePort{ - { - Name: "5432", - Port: 5432, - TargetPort: intstr.FromInt32(5432), - }, - { - Name: "metrics", - Port: 2112, - TargetPort: intstr.FromInt32(2112), - }, - }, - }, - }, - } -} - func newPostgresSession(t *testing.T, ctx context.Context) *sql.DB { var db *sql.DB err := retry.Do(func() error { - connString := fmt.Sprintf("host=127.0.0.1 port=5432 user=%s password=%s dbname=%s sslmode=disable", postgresUser, postgresPassword, postgresDB) + connString := fmt.Sprintf("host=127.0.0.1 port=5432 user=%s password=%s dbname=%s sslmode=disable", examples.PostgresUser, examples.PostgresPassword, examples.PostgresDB) var err error db, err = sql.Open("postgres", connString) diff --git a/integration/rethinkdb_test.go b/integration/rethinkdb_test.go index 69b164c..c330ff9 100644 --- a/integration/rethinkdb_test.go +++ b/integration/rethinkdb_test.go @@ -7,348 +7,34 @@ import ( "fmt" "testing" - "github.com/metal-stack/backup-restore-sidecar/pkg/constants" - "github.com/metal-stack/metal-lib/pkg/pointer" + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/avast/retry-go/v4" r "gopkg.in/rethinkdb/rethinkdb-go.v6" ) -type rethinkDbTestData struct { - ID string `rethinkdb:"id"` - Data string `rethinkdb:"data"` -} - const ( - rethinkDbPassword = "test123!" rethinkDbDatabaseName = "backup-restore" rethinkDbTable = "precioustestdata" ) -var ( - rethinkDbContainerImage = "rethinkdb:2.4.0" -) +type rethinkDbTestData struct { + ID string `rethinkdb:"id"` + Data string `rethinkdb:"data"` +} func Test_RethinkDB_Restore(t *testing.T) { restoreFlow(t, &flowSpec{ - databaseType: "rethinkdb", - sts: rethinkDbSts, - backingResources: rethinkDbBackingResources, + databaseType: examples.RethinkDB, + sts: examples.RethinkDbSts, + backingResources: examples.RethinkDbBackingResources, addTestData: addRethinkDbTestData, verifyTestData: verifyRethinkDbTestData, }) } -func rethinkDbSts(namespace, image string) *appsv1.StatefulSet { - if image == "" { - image = rethinkDbContainerImage - } - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "StatefulSet", - APIVersion: appsv1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rethinkdb", - Namespace: namespace, - Labels: map[string]string{ - "app": "rethinkdb", - }, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: "rethinkdb", - Replicas: pointer.Pointer(int32(1)), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "rethinkdb", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "rethinkdb", - }, - }, - Spec: corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{ - { - Name: "rethinkdb", - Image: image, - Command: []string{"backup-restore-sidecar", "wait"}, - Env: []corev1.EnvVar{ - { - Name: "RETHINKDB_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "rethinkdb", - }, - Key: "rethinkdb-password", - }, - }, - }, - }, - Ports: []corev1.ContainerPort{ - { - ContainerPort: 8080, - }, - { - ContainerPort: 28015, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/data", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - }, - }, - { - Name: "backup-restore-sidecar", - Image: image, - Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 8000, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "backup", - MountPath: constants.SidecarBaseDir, - }, - { - Name: "data", - MountPath: "/data", - }, - { - Name: "rethinkdb-credentials", - MountPath: "/rethinkdb-secret", - }, - { - Name: "backup-restore-sidecar-config", - MountPath: "/etc/backup-restore-sidecar", - }, - { - Name: "bin-provision", - SubPath: "backup-restore-sidecar", - MountPath: "/usr/local/bin/backup-restore-sidecar", - }, - { - Name: "bin-provision", - SubPath: "rethinkdb-dump", - MountPath: "/usr/local/bin/rethinkdb-dump", - }, - { - Name: "bin-provision", - SubPath: "rethinkdb-restore", - MountPath: "/usr/local/bin/rethinkdb-restore", - }, - }, - }, - }, - InitContainers: []corev1.Container{ - { - Name: "backup-restore-sidecar-provider", - Image: backupRestoreSidecarContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "cp", - "/backup-restore-sidecar", - "/rethinkdb/rethinkdb-dump", - "/rethinkdb/rethinkdb-restore", - "/bin-provision", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "bin-provision", - MountPath: "/bin-provision", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "data", - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "backup", - }, - }, - }, - { - Name: "rethinkdb-credentials", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "rethinkdb", - Items: []corev1.KeyToPath{ - { - Key: "rethinkdb-password", - Path: "rethinkdb-password.txt", - }, - }, - }, - }, - }, - { - Name: "backup-restore-sidecar-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "backup-restore-sidecar-config-rethinkdb", - }, - }, - }, - }, - { - Name: "bin-provision", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "data", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - }, - }, - } -} - -func rethinkDbBackingResources(namespace string) []client.Object { - return []client.Object{ - &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - Kind: "ConfigMap", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup-restore-sidecar-config-rethinkdb", - Namespace: namespace, - }, - Data: map[string]string{ - "config.yaml": `--- -bind-addr: 0.0.0.0 -db: rethinkdb -db-data-directory: /data/rethinkdb/ -backup-provider: local -rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt -backup-cron-schedule: "*/1 * * * *" -object-prefix: rethinkdb-test -post-exec-cmds: -# IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location -- rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} -`, - }, - }, - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rethinkdb", - Namespace: namespace, - }, - StringData: map[string]string{ - "rethinkdb-password": rethinkDbPassword, - }, - }, - &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rethinkdb", - Namespace: namespace, - Labels: map[string]string{ - "app": "rethinkdb", - }, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "rethinkdb", - }, - Ports: []corev1.ServicePort{ - { - Name: "10080", - Port: 10080, - TargetPort: intstr.FromInt32(10080), - }, - { - Name: "28015", - Port: 28015, - TargetPort: intstr.FromInt32(28015), - }, - { - Name: "metrics", - Port: 2112, - TargetPort: intstr.FromInt32(2112), - }, - }, - }, - }, - } -} - func newRethinkdbSession(t *testing.T, ctx context.Context) *r.Session { var session *r.Session err := retry.Do(func() error { @@ -357,7 +43,7 @@ func newRethinkdbSession(t *testing.T, ctx context.Context) *r.Session { Addresses: []string{"localhost:28015"}, Database: rethinkDbDatabaseName, Username: "admin", - Password: rethinkDbPassword, + Password: examples.RethinkDbPassword, MaxIdle: 10, MaxOpen: 20, }) diff --git a/kind.yaml b/kind.yaml index 02520dc..eaed97e 100644 --- a/kind.yaml +++ b/kind.yaml @@ -16,6 +16,9 @@ nodes: - containerPort: 5432 hostPort: 5432 listenAddress: 0.0.0.0 + - containerPort: 7700 + hostPort: 7700 + listenAddress: 0.0.0.0 - containerPort: 32379 hostPort: 32379 listenAddress: 0.0.0.0 diff --git a/pkg/generate/examples/dump.go b/pkg/generate/examples/dump.go new file mode 100644 index 0000000..cc774a1 --- /dev/null +++ b/pkg/generate/examples/dump.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "path" + "runtime" + + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" + appsv1 "k8s.io/api/apps/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +func main() { + for _, localExample := range []struct { + db string + sts func(namespace string) *appsv1.StatefulSet + backing func(namespace string) []client.Object + }{ + { + db: examples.Etcd, + sts: examples.EtcdSts, + backing: examples.EtcdBackingResources, + }, + { + db: examples.Meilisearch, + sts: examples.MeilisearchSts, + backing: examples.MeilisearchBackingResources, + }, + { + db: examples.Postgres, + sts: examples.PostgresSts, + backing: examples.PostgresBackingResources, + }, + { + db: examples.RethinkDB, + sts: examples.RethinkDbSts, + backing: examples.RethinkDbBackingResources, + }, + } { + err := dumpToExamples(localExample.db+"-local.yaml", append([]client.Object{localExample.sts("default")}, localExample.backing("default")...)...) + if err != nil { + panic(err) + } + } + + // TODO: add backup provider examples +} + +func dumpToExamples(name string, resources ...client.Object) error { + content := []byte(`# THESE EXAMPLES ARE GENERATED! +# Use them as a template for your deployment, but do not commit manual changes to these files. +--- +`) + + for i, r := range resources { + r.SetNamespace("") // not needed for example manifests + + r := r.DeepCopyObject() + + if sts, ok := r.(*appsv1.StatefulSet); ok { + // host network is only for integration testing purposes + sts.Spec.Template.Spec.HostNetwork = false + } + + raw, err := yaml.Marshal(r) + if err != nil { + return err + } + + if i != len(resources)-1 { + raw = append(raw, []byte("---\n")...) + } + + content = append(content, raw...) + } + + _, filename, _, _ := runtime.Caller(1) + + dest := path.Join(path.Dir(filename), "../../..", "deploy", name) + fmt.Printf("example manifest written to %s\n", dest) + + err := os.WriteFile(dest, content, 0600) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/generate/examples/examples/common.go b/pkg/generate/examples/examples/common.go new file mode 100644 index 0000000..e843ba5 --- /dev/null +++ b/pkg/generate/examples/examples/common.go @@ -0,0 +1,5 @@ +package examples + +const ( + backupRestoreSidecarContainerImage = "ghcr.io/metal-stack/backup-restore-sidecar:latest" +) diff --git a/pkg/generate/examples/examples/etcd.go b/pkg/generate/examples/examples/etcd.go new file mode 100644 index 0000000..77ddbd6 --- /dev/null +++ b/pkg/generate/examples/examples/etcd.go @@ -0,0 +1,255 @@ +package examples + +import ( + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "github.com/metal-stack/metal-lib/pkg/pointer" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + Etcd = "etcd" + etcdContainerImage = "quay.io/coreos/etcd:v3.5.7" +) + +func EtcdSts(namespace string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd", + Namespace: namespace, + Labels: map[string]string{ + "app": "etcd", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "etcd", + Replicas: pointer.Pointer(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "etcd", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "etcd", + Image: etcdContainerImage, + Command: []string{"backup-restore-sidecar", "wait"}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/usr/local/bin/etcdctl", "endpoint", "health", "--endpoints=127.0.0.1:32379"}, + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt(32381), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Ports: []corev1.ContainerPort{ + // default ports are taken by kind etcd because running in host network + { + ContainerPort: 32379, + Name: "client", + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 32380, + Name: "server", + Protocol: corev1.ProtocolTCP, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + }, + }, + { + Name: "backup-restore-sidecar", + Image: etcdContainerImage, + Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 8000, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "backup", + MountPath: constants.SidecarBaseDir, + }, + { + Name: "data", + MountPath: "/data", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "backup-restore-sidecar-provider", + Image: backupRestoreSidecarContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "cp", + "/backup-restore-sidecar", + "/bin-provision", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "bin-provision", + MountPath: "/bin-provision", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "data", + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup", + }, + }, + }, + { + Name: "backup-restore-sidecar-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "backup-restore-sidecar-config-postgres", + }, + }, + }, + }, + { + Name: "bin-provision", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func EtcdBackingResources(namespace string) []client.Object { + return []client.Object{ + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-restore-sidecar-config-postgres", + Namespace: namespace, + }, + Data: map[string]string{ + "config.yaml": `--- +bind-addr: 0.0.0.0 +db: etcd +db-data-directory: /data/etcd/ +backup-provider: local +backup-cron-schedule: "*/1 * * * *" +object-prefix: etcd-test +etcd-endpoints: http://localhost:32379 +post-exec-cmds: +- etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 +`, + }, + }, + } +} diff --git a/pkg/generate/examples/examples/meilisearch.go b/pkg/generate/examples/examples/meilisearch.go new file mode 100644 index 0000000..75ea16d --- /dev/null +++ b/pkg/generate/examples/examples/meilisearch.go @@ -0,0 +1,332 @@ +package examples + +import ( + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "github.com/metal-stack/metal-lib/pkg/pointer" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + Meilisearch = "meilisearch" + MeilisearchPassword = "test123!" + + meilisearchContainerImage = "getmeili/meilisearch:v1.3.0" +) + +func MeilisearchSts(namespace string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "meilisearch", + Namespace: namespace, + Labels: map[string]string{ + "app": "meilisearch", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "meilisearch", + Replicas: pointer.Pointer(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "meilisearch", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "meilisearch", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "meilisearch", + Image: meilisearchContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"backup-restore-sidecar", "wait"}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + PeriodSeconds: 10, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromString("http"), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + TimeoutSeconds: 10, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 7700, + Name: "http", + Protocol: corev1.ProtocolTCP, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "backup", + MountPath: constants.SidecarBaseDir, + }, + { + Name: "data", + MountPath: "/data", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + }, + }, + { + Name: "backup-restore-sidecar", + Image: meilisearchContainerImage, + Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, + Env: []corev1.EnvVar{ + { + Name: "BACKUP_RESTORE_SIDECAR_MEILISEARCH_APIKEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "meilisearch", + }, + Key: "MEILISEARCH_APIKEY", + }, + }, + }, + { + Name: "BACKUP_RESTORE_SIDECAR_MEILISEARCH_URL", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "meilisearch", + }, + Key: "MEILISEARCH_URL", + }, + }}, + }, + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 8000, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "backup", + MountPath: constants.SidecarBaseDir, + }, + { + Name: "data", + MountPath: "/data", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "backup-restore-sidecar-provider", + Image: backupRestoreSidecarContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "cp", + "/backup-restore-sidecar", + "/bin-provision", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "bin-provision", + MountPath: "/bin-provision", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "data", + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup", + }, + }, + }, + { + Name: "backup-restore-sidecar-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "backup-restore-sidecar-config-meilisearch", + }, + }, + }, + }, + { + Name: "bin-provision", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func MeilisearchBackingResources(namespace string) []client.Object { + return []client.Object{ + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-restore-sidecar-config-meilisearch", + Namespace: namespace, + }, + Data: map[string]string{ + "config.yaml": `--- +bind-addr: 0.0.0.0 +db: meilisearch +db-data-directory: /data/data.ms/ +backup-provider: local +backup-cron-schedule: "*/1 * * * *" +object-prefix: meilisearch-test +compression-method: targz +post-exec-cmds: +- meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files +`, + }, + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "meilisearch", + Namespace: namespace, + }, + StringData: map[string]string{ + "MEILISEARCH_APIKEY": MeilisearchPassword, + "MEILISEARCH_URL": "http://localhost:7700", + }, + }, + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "meilisearch", + Namespace: namespace, + Labels: map[string]string{ + "app": "meilisearch", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "meilisearch", + }, + Ports: []corev1.ServicePort{ + { + Name: "7700", + Port: 7700, + TargetPort: intstr.FromInt32(7700), + }, + { + Name: "metrics", + Port: 2112, + TargetPort: intstr.FromInt32(2112), + }, + }, + }, + }, + } +} diff --git a/pkg/generate/examples/examples/postgres.go b/pkg/generate/examples/examples/postgres.go new file mode 100644 index 0000000..8b75f91 --- /dev/null +++ b/pkg/generate/examples/examples/postgres.go @@ -0,0 +1,364 @@ +package examples + +import ( + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "github.com/metal-stack/metal-lib/pkg/pointer" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + Postgres = "postgres" + + PostgresDB = "postgres" + PostgresPassword = "test123!" + PostgresUser = "postgres" + + postgresContainerImage = "postgres:12-alpine" +) + +func PostgresSts(namespace string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres", + Namespace: namespace, + Labels: map[string]string{ + "app": "postgres", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "postgres", + Replicas: pointer.Pointer(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "postgres", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "postgres", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "postgres", + Image: postgresContainerImage, + Command: []string{"backup-restore-sidecar", "wait"}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "exec", "pg_isready", "-U", PostgresUser, "-h", "127.0.0.1", "-p", "5432"}, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "exec", "pg_isready", "-U", PostgresUser, "-h", "127.0.0.1", "-p", "5432"}, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 5, + PeriodSeconds: 10, + }, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_DB", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_DB", + }, + }, + }, + { + Name: "POSTGRES_USER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_USER", + }, + }, + }, + { + Name: "POSTGRES_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_PASSWORD", + }, + }, + }, + { + Name: "PGDATA", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_DATA", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 5432, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + }, + }, + { + Name: "backup-restore-sidecar", + Image: postgresContainerImage, + Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, + Env: []corev1.EnvVar{ + { + Name: "BACKUP_RESTORE_SIDECAR_POSTGRES_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_PASSWORD", + }, + }, + }, + { + Name: "BACKUP_RESTORE_SIDECAR_POSTGRES_USER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "postgres", + }, + Key: "POSTGRES_USER", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 8000, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "backup", + MountPath: constants.SidecarBaseDir, + }, + { + Name: "data", + MountPath: "/data", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "backup-restore-sidecar-provider", + Image: backupRestoreSidecarContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "cp", + "/backup-restore-sidecar", + "/bin-provision", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "bin-provision", + MountPath: "/bin-provision", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "data", + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup", + }, + }, + }, + { + Name: "backup-restore-sidecar-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "backup-restore-sidecar-config-postgres", + }, + }, + }, + }, + { + Name: "bin-provision", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func PostgresBackingResources(namespace string) []client.Object { + return []client.Object{ + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-restore-sidecar-config-postgres", + Namespace: namespace, + }, + Data: map[string]string{ + "config.yaml": `--- +bind-addr: 0.0.0.0 +db: postgres +db-data-directory: /data/postgres/ +backup-provider: local +backup-cron-schedule: "*/1 * * * *" +object-prefix: postgres-test +compression-method: tar +post-exec-cmds: +- docker-entrypoint.sh postgres +`, + }, + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres", + Namespace: namespace, + }, + StringData: map[string]string{ + "POSTGRES_DB": PostgresDB, + "POSTGRES_USER": PostgresUser, + "POSTGRES_PASSWORD": PostgresPassword, + "POSTGRES_DATA": "/data/postgres/", + }, + }, + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres", + Namespace: namespace, + Labels: map[string]string{ + "app": "postgres", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "postgres", + }, + Ports: []corev1.ServicePort{ + { + Name: "5432", + Port: 5432, + TargetPort: intstr.FromInt32(5432), + }, + { + Name: "metrics", + Port: 2112, + TargetPort: intstr.FromInt32(2112), + }, + }, + }, + }, + } +} diff --git a/pkg/generate/examples/examples/rethinkdb.go b/pkg/generate/examples/examples/rethinkdb.go new file mode 100644 index 0000000..57de656 --- /dev/null +++ b/pkg/generate/examples/examples/rethinkdb.go @@ -0,0 +1,319 @@ +package examples + +import ( + "github.com/metal-stack/backup-restore-sidecar/pkg/constants" + "github.com/metal-stack/metal-lib/pkg/pointer" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + RethinkDB = "rethinkdb" + + RethinkDbPassword = "test123!" + + rethinkDbContainerImage = "rethinkdb:2.4.0" +) + +func RethinkDbSts(namespace string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rethinkdb", + Namespace: namespace, + Labels: map[string]string{ + "app": "rethinkdb", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "rethinkdb", + Replicas: pointer.Pointer(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "rethinkdb", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "rethinkdb", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "rethinkdb", + Image: rethinkDbContainerImage, + Command: []string{"backup-restore-sidecar", "wait"}, + Env: []corev1.EnvVar{ + { + Name: "RETHINKDB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "rethinkdb", + }, + Key: "rethinkdb-password", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8080, + }, + { + ContainerPort: 28015, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + }, + }, + { + Name: "backup-restore-sidecar", + Image: rethinkDbContainerImage, + Command: []string{"backup-restore-sidecar", "start", "--log-level=debug"}, + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 8000, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "backup", + MountPath: constants.SidecarBaseDir, + }, + { + Name: "data", + MountPath: "/data", + }, + { + Name: "rethinkdb-credentials", + MountPath: "/rethinkdb-secret", + }, + { + Name: "backup-restore-sidecar-config", + MountPath: "/etc/backup-restore-sidecar", + }, + { + Name: "bin-provision", + SubPath: "backup-restore-sidecar", + MountPath: "/usr/local/bin/backup-restore-sidecar", + }, + { + Name: "bin-provision", + SubPath: "rethinkdb-dump", + MountPath: "/usr/local/bin/rethinkdb-dump", + }, + { + Name: "bin-provision", + SubPath: "rethinkdb-restore", + MountPath: "/usr/local/bin/rethinkdb-restore", + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "backup-restore-sidecar-provider", + Image: backupRestoreSidecarContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "cp", + "/backup-restore-sidecar", + "/rethinkdb/rethinkdb-dump", + "/rethinkdb/rethinkdb-restore", + "/bin-provision", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "bin-provision", + MountPath: "/bin-provision", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "data", + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup", + }, + }, + }, + { + Name: "rethinkdb-credentials", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "rethinkdb", + Items: []corev1.KeyToPath{ + { + Key: "rethinkdb-password", + Path: "rethinkdb-password.txt", + }, + }, + }, + }, + }, + { + Name: "backup-restore-sidecar-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "backup-restore-sidecar-config-rethinkdb", + }, + }, + }, + }, + { + Name: "bin-provision", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func RethinkDbBackingResources(namespace string) []client.Object { + return []client.Object{ + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-restore-sidecar-config-rethinkdb", + Namespace: namespace, + }, + Data: map[string]string{ + "config.yaml": `--- +bind-addr: 0.0.0.0 +db: rethinkdb +db-data-directory: /data/rethinkdb/ +backup-provider: local +rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt +backup-cron-schedule: "*/1 * * * *" +object-prefix: rethinkdb-test +post-exec-cmds: +# IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location +- rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} +`, + }, + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rethinkdb", + Namespace: namespace, + }, + StringData: map[string]string{ + "rethinkdb-password": RethinkDbPassword, + }, + }, + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rethinkdb", + Namespace: namespace, + Labels: map[string]string{ + "app": "rethinkdb", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "rethinkdb", + }, + Ports: []corev1.ServicePort{ + { + Name: "10080", + Port: 10080, + TargetPort: intstr.FromInt32(10080), + }, + { + Name: "28015", + Port: 28015, + TargetPort: intstr.FromInt32(28015), + }, + { + Name: "metrics", + Port: 2112, + TargetPort: intstr.FromInt32(2112), + }, + }, + }, + }, + } +}