Skip to content

Commit

Permalink
Another review with some improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
Gerrit91 committed Sep 19, 2023
1 parent a384aeb commit 9390f6f
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 38 deletions.
2 changes: 1 addition & 1 deletion cmd/internal/database/etcd/etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func New(log *zap.SugaredLogger, datadir, caCert, cert, key, endpoints, name str
}
}

// Check checks whether a backup needs to be restored or not, returns true if it needs a backup
// Check indicates whether a restore of the database is required or not.
func (db *Etcd) Check(_ context.Context) (bool, error) {
empty, err := utils.IsEmpty(db.datadir)
if err != nil {
Expand Down
41 changes: 24 additions & 17 deletions cmd/internal/database/meilisearch/meilisearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"syscall"
"time"

"github.com/avast/retry-go/v4"
"github.com/meilisearch/meilisearch-go"
"github.com/spf13/afero"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -83,10 +82,15 @@ func (db *Meilisearch) Backup(ctx context.Context) error {
if err != nil {
return fmt.Errorf("unable to find dump: %w", err)
}
if len(dumps) == 0 {
if len(dumps) != 1 {
return fmt.Errorf("did not find unique dump, found %d", len(dumps))
}

// we need to do a copy here and cannot simply rename as the file system is
// mounted by two containers. the dump is created in the database container,
// the copy is done in the backup-restore-sidecar container. os.Rename would
// lead to an error.

err = utils.Copy(afero.NewOsFs(), dumps[0], path.Join(constants.BackupDir, latestStableDump))
if err != nil {
return fmt.Errorf("unable to move dump to latest: %w", err)
Expand All @@ -110,7 +114,7 @@ func (db *Meilisearch) Backup(ctx context.Context) error {
return nil
}

// Check checks whether a backup needs to be restored or not, returns true if it needs a backup
// Check indicates whether a restore of the database is required or not.
func (db *Meilisearch) Check(_ context.Context) (bool, error) {
empty, err := utils.IsEmpty(db.datadir)
if err != nil {
Expand Down Expand Up @@ -159,7 +163,7 @@ func (db *Meilisearch) Recover(ctx context.Context) error {
return fmt.Errorf("unable to recover %w", err)
}

db.log.Infow("recovery done", "duration", time.Since(start))
db.log.Infow("successfully restored meilisearch database", "duration", time.Since(start).String())

return nil
}
Expand Down Expand Up @@ -195,29 +199,32 @@ func (db *Meilisearch) importDump(ctx context.Context, dump string) error {
return err
}

db.log.Info("import of dump finished")
db.log.Info("execution of meilisearch finished without an error")

return nil
})

// TODO big databases might take longer, not sure if 100 attempts are enough
// must check how long it take max with backoff ?
err = retry.Do(func() error {
restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey)
if err != nil {
return fmt.Errorf("unable to create prober")
}
restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey)
if err != nil {
return fmt.Errorf("unable to create prober")
}

for {
time.Sleep(3 * time.Second)

err = restoreDB.Probe(ctx)
if err != nil {
db.log.Errorw("meilisearch is still restoring, continue probing for readiness...", "error", err)

return err
}
} else {
db.log.Infow("meilisearch started after importing the dump, stopping it again for takeover from the database container")

db.log.Infow("meilisearch started after importing the dump, stopping it again")
break
}
}

return cmd.Process.Signal(syscall.SIGINT)
}, retry.Attempts(100), retry.Context(ctx))
if err != nil {
if err := cmd.Process.Signal(syscall.SIGINT); err != nil {
return handleFailedRecovery(err)
}

Expand Down
52 changes: 34 additions & 18 deletions cmd/internal/database/meilisearch/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ func (db *Meilisearch) Upgrade(ctx context.Context) error {
if err != nil {
return err
}

meilisearchVersion, err := db.getBinaryVersion(ctx)
if err != nil {
return err
db.log.Errorw("unable to get binary version, skipping upgrade", "error", err)
return nil
}

if dbVersion.String() == meilisearchVersion.String() {
db.log.Infow("no version difference, no upgrade required", "database-version", dbVersion, "binary-version", meilisearchVersion)
return nil
Expand All @@ -55,29 +58,29 @@ func (db *Meilisearch) Upgrade(ctx context.Context) error {

err = db.dumpWithOldBinary(ctx)
if err != nil {
return err
return fmt.Errorf("unable to create dump with old meilisearch binary: %w", err)
}

oldVersionDataDir := strings.TrimRight(db.datadir, "/") + ".upgrade"

err = os.Rename(db.datadir, oldVersionDataDir)
if err != nil {
return fmt.Errorf("cannot move old version data dir out of the way: %w", err)
return fmt.Errorf("cannot move old version data dir out of the way, which could have happened due to a failed recovery attempt, consider manual cleanup: %w", err)
}

dump := path.Join(constants.BackupDir, latestStableDump)

err = db.importDump(ctx, dump)
if err != nil {
return err
return fmt.Errorf("unable to import dump with new meilisearch binary: %w", err)
}

err = os.RemoveAll(oldVersionDataDir)
if err != nil {
return fmt.Errorf("unable cleanup old version data dir: %w", err)
db.log.Errorw("unable cleanup old version data dir, consider manual cleanup", "error", err)
}

db.log.Infow("upgrade done and new data in place", "took", time.Since(start))
db.log.Infow("meilisearch upgrade done and new data in place", "duration", time.Since(start))

return nil
}
Expand Down Expand Up @@ -136,6 +139,7 @@ func (db *Meilisearch) getDatabaseVersion(versionFile string) (*semver.Version,
if err != nil {
return nil, fmt.Errorf("unable to parse meilisearch binary version in %q: %w", string(versionBytes), err)
}

return v, nil
}

Expand Down Expand Up @@ -183,27 +187,39 @@ func (db *Meilisearch) dumpWithOldBinary(ctx context.Context) error {
return nil
})

err = retry.Do(func() error {
restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey)
if err != nil {
return fmt.Errorf("unable to create prober")
}
restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey)
if err != nil {
return fmt.Errorf("unable to create prober")
}

restoreDB.copyBinaryAfterBackup = false
restoreDB.copyBinaryAfterBackup = false

err = restoreDB.Backup(ctx)
err = retry.Do(func() error {
err = restoreDB.Probe(ctx)
if err != nil {
return fmt.Errorf("unable to create dump from previous meilisearch version")
}
db.log.Errorw("meilisearch is still starting, continue probing for readiness...", "error", err)

db.log.Infow("taken dump from previous meilisearch version, stopping it again")
return err
}

return cmd.Process.Signal(syscall.SIGINT)
db.log.Infow("previous meilisearch started and is now ready for backup")
return nil
}, retry.Context(ctx))
if err != nil {
return err
}

err = restoreDB.Backup(ctx)
if err != nil {
return fmt.Errorf("unable to create dump from previous meilisearch version")
}

db.log.Infow("taken dump from previous meilisearch version, stopping it again")

if err := cmd.Process.Signal(syscall.SIGINT); err != nil {
return err
}

err = g.Wait()
if err != nil {
// will probably work better in meilisearch v1.4.0: https://github.com/meilisearch/meilisearch/commit/eff8570f591fe32a6106087807e3fe8c18e8e5e4
Expand All @@ -214,7 +230,7 @@ func (db *Meilisearch) dumpWithOldBinary(ctx context.Context) error {
}
}

db.log.Info("successfully took dump from previous meilisearch database")
db.log.Info("successfully took dump with previous meilisearch version")

return nil
}
2 changes: 1 addition & 1 deletion cmd/internal/database/postgres/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func New(log *zap.SugaredLogger, datadir string, host string, port int, user str
}
}

// Check checks whether a backup needs to be restored or not, returns true if it needs a backup
// Check indicates whether a restore of the database is required or not.
func (db *Postgres) Check(_ context.Context) (bool, error) {
empty, err := utils.IsEmpty(db.datadir)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/internal/database/rethinkdb/rethinkdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func New(log *zap.SugaredLogger, datadir string, url string, passwordFile string
}
}

// Check checks whether a backup needs to be restored or not, returns true if it needs a backup
// Check indicates whether a restore of the database is required or not.
func (db *RethinkDB) Check(_ context.Context) (bool, error) {
empty, err := utils.IsEmpty(db.datadir)
if err != nil {
Expand Down

0 comments on commit 9390f6f

Please sign in to comment.