Initial Meilisearch Support

.github/workflows/docker.yaml

@@ -70,7 +70,7 @@ jobs:
     - name: Lint
       uses: golangci/golangci-lint-action@v3
       with:
-        args: --build-tags integration -p bugs -p unused --timeout=3m
+        args: --build-tags integration -p bugs -p unused --timeout=10m
 
     - name: Test
       run: |

Makefile

@@ -22,7 +22,7 @@ GO_RUN_ARG := -run $(GO_RUN)
 endif
 
 .PHONY: build
-build:
+build: generate-examples
 	go mod tidy
 	go build -ldflags "$(LINKMODE)" -tags 'osusergo netgo static_build' -o bin/backup-restore-sidecar github.com/metal-stack/backup-restore-sidecar/cmd
 	strip bin/backup-restore-sidecar
@@ -31,10 +31,14 @@ build:
 test: build
 	go test -cover ./...
 
+.PHONY: generate-examples
+generate-examples:
+	go run ./pkg/generate/examples/dump.go
+
 .PHONY: test-integration
 test-integration: kind-cluster-create
 	kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest
-	KUBECONFIG=$(KUBECONFIG) go test $(GO_RUN_ARG) -tags=integration -count 1 -v -p 1 -timeout 10m ./...
+	KUBECONFIG=$(KUBECONFIG) go test $(GO_RUN_ARG) -tags=integration -count 1 -v -p 1 -timeout 20m ./...
 
 .PHONY: proto
 proto:
@@ -60,6 +64,10 @@ start-rethinkdb:
 start-etcd:
 	$(MAKE)	start	DB=etcd
 
+.PHONY: start-meilisearch
+start-meilisearch:
+	$(MAKE)	start	DB=meilisearch
+
 .PHONY: start
 start: kind-cluster-create
 	kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest

README.md

@@ -8,11 +8,12 @@ Probably, it does not make sense to use this project with large databases. Howev
 
 ## Supported Databases
 
-| Database  | Image        | Status | Upgrade Support |
-| --------- | ------------ | :----: | :-------------: |
-| postgres  | >= 12-alpine |  beta  |       ✅        |
-| rethinkdb | >= 2.4.0     |  beta  |       ❌        |
-| ETCD      | >= 3.5       | alpha  |       ❌        |
+| Database    | Image        | Status | Upgrade Support |
+| ----------- | ------------ | :----: | :-------------: |
+| postgres    | >= 12-alpine |  beta  |       ✅        |
+| rethinkdb   | >= 2.4.0     |  beta  |       ❌        |
+| ETCD        | >= 3.5       | alpha  |       ❌        |
+| meilisearch | >= 1.2.0     | alpha  |       ✅        |
 
 ## Database Upgrades
 

cmd/internal/database/etcd/etcd.go

@@ -43,7 +43,7 @@ func New(log *zap.SugaredLogger, datadir, caCert, cert, key, endpoints, name str
 	}
 }
 
-// Check checks whether a backup needs to be restored or not, returns true if it needs a backup
+// Check indicates whether a restore of the database is required or not.
 func (db *Etcd) Check(_ context.Context) (bool, error) {
 	empty, err := utils.IsEmpty(db.datadir)
 	if err != nil {

cmd/internal/database/meilisearch/meilisearch.go

@@ -0,0 +1,269 @@
+package meilisearch
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/meilisearch/meilisearch-go"
+	"github.com/spf13/afero"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+
+	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils"
+	"github.com/metal-stack/backup-restore-sidecar/pkg/constants"
+	"go.uber.org/zap"
+)
+
+const (
+	meilisearchCmd         = "meilisearch"
+	meilisearchVersionFile = "VERSION"
+	meilisearchDBDir       = "data.ms"
+	latestStableDump       = "latest.dump"
+)
+
+// Meilisearch implements the database interface
+type Meilisearch struct {
+	log                   *zap.SugaredLogger
+	executor              *utils.CmdExecutor
+	datadir               string
+	copyBinaryAfterBackup bool
+
+	apikey string
+	client *meilisearch.Client
+}
+
+// New instantiates a new meilisearch database
+func New(log *zap.SugaredLogger, datadir string, url string, apikey string) (*Meilisearch, error) {
+	if url == "" {
+		return nil, fmt.Errorf("meilisearch api url cannot be empty")
+	}
+	if apikey == "" {
+		return nil, fmt.Errorf("meilisearch api key cannot be empty")
+	}
+
+	client := meilisearch.NewClient(meilisearch.ClientConfig{
+		Host:   url,
+		APIKey: apikey,
+	})
+
+	return &Meilisearch{
+		log:                   log,
+		datadir:               datadir,
+		apikey:                apikey,
+		executor:              utils.NewExecutor(log),
+		client:                client,
+		copyBinaryAfterBackup: true,
+	}, nil
+}
+
+// Backup takes a dump of meilisearch with the meilisearch client.
+func (db *Meilisearch) Backup(ctx context.Context) error {
+	if err := os.RemoveAll(constants.BackupDir); err != nil {
+		return fmt.Errorf("could not clean backup directory: %w", err)
+	}
+
+	if err := os.MkdirAll(constants.BackupDir, 0777); err != nil {
+		return fmt.Errorf("could not create backup directory: %w", err)
+	}
+
+	dumpResponse, err := db.client.CreateDump()
+	if err != nil {
+		return fmt.Errorf("could not create a dump: %w", err)
+	}
+
+	db.log.Infow("dump creation triggered", "taskUUID", dumpResponse.TaskUID)
+
+	dumpTask, err := db.client.WaitForTask(dumpResponse.TaskUID, meilisearch.WaitParams{Context: ctx})
+	if err != nil {
+		return err
+	}
+	db.log.Infow("dump created successfully", "duration", dumpTask.Duration)
+
+	dumps, err := filepath.Glob(constants.BackupDir + "/*.dump")
+	if err != nil {
+		return fmt.Errorf("unable to find dump: %w", err)
+	}
+	if len(dumps) != 1 {
+		return fmt.Errorf("did not find unique dump, found %d", len(dumps))
+	}
+
+	// we need to do a copy here and cannot simply rename as the file system is
+	// mounted by two containers. the dump is created in the database container,
+	// the copy is done in the backup-restore-sidecar container. os.Rename would
+	// lead to an error.
+
+	err = utils.Copy(afero.NewOsFs(), dumps[0], path.Join(constants.BackupDir, latestStableDump))
+	if err != nil {
+		return fmt.Errorf("unable to move dump to latest: %w", err)
+	}
+
+	err = os.Remove(dumps[0])
+	if err != nil {
+		return fmt.Errorf("unable to clean up dump: %w", err)
+	}
+
+	db.log.Debugw("successfully took backup of meilisearch")
+
+	if db.copyBinaryAfterBackup {
+		// for a future upgrade, the current meilisearch binary is required
+		err = db.copyMeilisearchBinary(ctx, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Check indicates whether a restore of the database is required or not.
+func (db *Meilisearch) Check(_ context.Context) (bool, error) {
+	empty, err := utils.IsEmpty(db.datadir)
+	if err != nil {
+		return false, err
+	}
+
+	if empty {
+		db.log.Info("data directory is empty")
+		return true, err
+	}
+
+	return false, nil
+}
+
+// Probe figures out if the database is running and available for taking backups.
+func (db *Meilisearch) Probe(_ context.Context) error {
+	_, err := db.client.Version()
+	if err != nil {
+		return fmt.Errorf("connection error: %w", err)
+	}
+
+	healthy := db.client.IsHealthy()
+	if !healthy {
+		return fmt.Errorf("meilisearch does not report healthiness")
+	}
+
+	return nil
+}
+
+// Recover restores a database backup
+func (db *Meilisearch) Recover(ctx context.Context) error {
+	dump := path.Join(constants.RestoreDir, latestStableDump)
+
+	if _, err := os.Stat(dump); os.IsNotExist(err) {
+		return fmt.Errorf("restore file not present: %s", dump)
+	}
+
+	if err := utils.RemoveContents(db.datadir); err != nil {
+		return fmt.Errorf("could not clean database data directory: %w", err)
+	}
+
+	start := time.Now()
+
+	err := db.importDump(ctx, dump)
+	if err != nil {
+		return fmt.Errorf("unable to recover %w", err)
+	}
+
+	db.log.Infow("successfully restored meilisearch database", "duration", time.Since(start).String())
+
+	return nil
+}
+
+func (db *Meilisearch) importDump(ctx context.Context, dump string) error {
+	var (
+		err  error
+		g, _ = errgroup.WithContext(ctx)
+
+		handleFailedRecovery = func(restoreErr error) error {
+			db.log.Errorw("trying to handle failed database recovery", "error", restoreErr)
+
+			if err := os.RemoveAll(db.datadir); err != nil {
+				db.log.Errorw("unable to cleanup database data directory after failed recovery attempt, high risk of starting with fresh database on container restart", "err", err)
+			} else {
+				db.log.Info("cleaned up database data directory after failed recovery attempt to prevent start of fresh database")
+			}
+
+			return restoreErr
+		}
+	)
+
+	args := []string{"--import-dump", dump, "--master-key", db.apikey, "--dump-dir", constants.RestoreDir, "--db-path", db.datadir, "--http-addr", "localhost:1"}
+	cmd := exec.CommandContext(ctx, meilisearchCmd, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	g.Go(func() error {
+		db.log.Infow("execute meilisearch", "args", args)
+
+		err = cmd.Run()
+		if err != nil {
+			return err
+		}
+
+		db.log.Info("execution of meilisearch finished without an error")
+
+		return nil
+	})
+
+	restoreDB, err := New(db.log, db.datadir, "http://localhost:1", db.apikey)
+	if err != nil {
+		return fmt.Errorf("unable to create prober")
+	}
+
+	waitForRestore := func() error {
+		ticker := time.NewTicker(3 * time.Second)
+		defer ticker.Stop()
+		sem := semaphore.NewWeighted(1)
+
+		for {
+			select {
+			case <-ticker.C:
+				if !sem.TryAcquire(1) {
+					continue
+				}
+
+				err = restoreDB.Probe(ctx)
+				sem.Release(1)
+				if err != nil {
+					db.log.Errorw("meilisearch is still restoring, continue probing for readiness...", "error", err)
+					continue
+				}
+
+				db.log.Infow("meilisearch started after importing the dump, stopping it again for takeover from the database container")
+
+				return nil
+			case <-ctx.Done():
+				return fmt.Errorf("context cancelled during meilisearch restore")
+			}
+		}
+	}
+
+	if err := waitForRestore(); err != nil {
+		return handleFailedRecovery(err)
+	}
+
+	if err := cmd.Process.Signal(syscall.SIGINT); err != nil {
+		return handleFailedRecovery(err)
+	}
+
+	err = g.Wait()
+	if err != nil {
+		// will probably work better in meilisearch v1.4.0: https://github.com/meilisearch/meilisearch/commit/eff8570f591fe32a6106087807e3fe8c18e8e5e4
+		if strings.Contains(err.Error(), "interrupt") {
+			db.log.Infow("meilisearch terminated but reported an error which can be ignored", "error", err)
+		} else {
+			return handleFailedRecovery(err)
+		}
+	}
+
+	db.log.Info("successfully restored meilisearch database")
+
+	return nil
+}