feat: support HashiCorp Vault with token based access (#38)

* feat: add source system support for hashicorp vault
* feat: implement cli flag for pointing to specific config file
* fix: wrong type in filesystem config

.editorconfig

@@ -13,3 +13,6 @@ trim_trailing_whitespace = true
 
 [{*.go,Makefile,go.mod,go.sum}]
 indent_style = tab
+
+[*.sh]
+indent_size = 4

cmd/backup.go

@@ -26,13 +26,15 @@ import (
 	"log/slog"
 )
 
+var cfgFilePath string
+
 // backupCmd represents the backup command
 var backupCmd = &cobra.Command{
 	Use:   "backup",
 	Short: "Execute the backup.",
 	Long:  "Execute the backup. Used config can be overridden by providing arguments.",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		c, err := config.New()
+		c, err := config.New(cfgFilePath)
 		if err != nil {
 			slog.Error("error while initializing config", "error", err)
 			return err
@@ -79,3 +81,13 @@ var backupCmd = &cobra.Command{
 		return nil
 	},
 }
+
+func init() {
+	backupCmd.PersistentFlags().StringVarP(
+		&cfgFilePath,
+		"config",
+		"c",
+		"",
+		"path to config file (default: ./config.yaml)",
+	)
+}

config.yaml

@@ -1,11 +1,21 @@
+# In order to enable environmental substitution, it is necessary to include all potential configurations in this
+# configuration file. Otherwise, the environment variables will not be automatically added to the ResourceConfig
+# (`mapstructure:",remain"`).
+
+loglevel: info
+
 source:
   type: filesystem
+
   filesystem:
     path: /tmp/source.txt
 
+  hashicorpvault:
+    token:
+    address:
+
 destination:
   type: filesystem
+
   filesystem:
     path: /tmp/destination.txt
-
-logLevel: debug

go.mod

@@ -3,17 +3,35 @@ module github.com/cluetec/lifeboat
 go 1.21
 
 require (
+	github.com/go-playground/validator/v10 v10.16.0
+	github.com/hashicorp/vault/api v1.10.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.17.0
 )
 
 require (
+	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.6 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -23,9 +41,12 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/config/config.go

@@ -63,8 +63,8 @@ func (c *Config) DebugEnabled() bool {
 	return strings.ToLower(c.LogLevel) == "debug"
 }
 
-func New() (*Config, error) {
-	if err := initViper(); err != nil {
+func New(cfgFilePath string) (*Config, error) {
+	if err := initViper(cfgFilePath); err != nil {
 		slog.Error("error while initializing viper", "error", err)
 		return nil, err
 	}

internal/config/viper.go

@@ -22,11 +22,16 @@ import (
 	"strings"
 )
 
-func initViper() error {
+func initViper(cfgFilePath string) error {
 	viper.AutomaticEnv()
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-	viper.AddConfigPath(".")
-	viper.SetConfigFile("./config.yaml")
+
+	if cfgFilePath == "" {
+		viper.AddConfigPath(".")
+		viper.SetConfigFile("./config.yaml")
+	} else {
+		viper.SetConfigFile(cfgFilePath)
+	}
 
 	if err := viper.ReadInConfig(); err != nil {
 		slog.Error("error while reading in the configs: %w", err)

internal/destination/filesystem/writer.go

@@ -30,18 +30,18 @@ type Writer struct {
 func NewWriter(rc *globalConfig.ResourceConfig) (*Writer, error) {
 	c, err := newConfig(rc)
 	if err != nil {
-		slog.Error("error while initializing filesystem destination config", "error", err)
+		slog.Error("error while initializing destination config", "destinationType", Type, "error", err)
 		return nil, err
 	}
 
-	slog.Debug("filesystem destination config loaded", "config", c)
+	slog.Debug("destination config loaded", "destinationType", Type, "config", c)
 
 	// Check if destination file already exists
 	_, err = os.Stat(c.Path)
 	if err == nil {
 		return nil, errors.New("destination file already exists")
 	} else if !errors.Is(err, os.ErrNotExist) {
-		slog.Error("error while checking if destination file already exists", "error", err)
+		slog.Error("error while checking if destination file already exists", "destinationType", Type, "error", err)
 		return nil, err
 	}
 
@@ -56,11 +56,12 @@ func NewWriter(rc *globalConfig.ResourceConfig) (*Writer, error) {
 
 func (w *Writer) Write(b []byte) (int, error) {
 	slog.Debug("filesystem destination write got called")
+	slog.Debug("write got called", "destinationType", Type)
 	return w.file.Write(b)
 }
 
 func (w *Writer) Close() error {
-	slog.Debug("closing filesystem writer")
+	slog.Debug("closing writer", "destinationType", Type)
 
 	if w.file != nil {
 		if err := w.file.Close(); err != nil {

internal/source/filesystem/config.go

@@ -25,7 +25,7 @@ import (
 const Type = "filesystem"
 
 type metaConfig struct {
-	Filesystem Config
+	Filesystem config
 }
 
 type config struct {

internal/source/filesystem/reader.go

@@ -29,11 +29,11 @@ type Reader struct {
 func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) {
 	c, err := newConfig(rc)
 	if err != nil {
-		slog.Error("error while initializing filesystem source config", "error", err)
+		slog.Error("error while initializing source config", "sourceType", Type, "error", err)
 		return nil, err
 	}
 
-	slog.Debug("filesystem source config loaded", "config", rc)
+	slog.Debug("source config loaded", "sourceType", Type, "config", c)
 
 	f, err := os.Open(c.Path)
 	if err != nil {
@@ -44,12 +44,12 @@ func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) {
 }
 
 func (r *Reader) Read(b []byte) (int, error) {
-	slog.Debug("filesystem source read got called")
+	slog.Debug("read got called", "sourceType", Type)
 	return r.file.Read(b)
 }
 
 func (r *Reader) Close() error {
-	slog.Debug("closing filesystem reader")
+	slog.Debug("closing reader", "sourceType", Type)
 
 	if r.file != nil {
 		if err := r.file.Close(); err != nil {

internal/source/hashicorpvault/config.go

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2023 cluetec GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hashicorpvault
+
+import (
+	globalConfig "github.com/cluetec/lifeboat/internal/config"
+	"github.com/go-playground/validator/v10"
+	vault "github.com/hashicorp/vault/api"
+	"github.com/mitchellh/mapstructure"
+	"log/slog"
+)
+
+const Type = "hashicorpvault"
+
+type metaConfig struct {
+	hashicorpvault config
+}
+
+type config struct {
+	Address string `validate:"http_url,required"`
+	Token   string `validate:"required"`
+}
+
+var validate *validator.Validate
+
+// newConfig provides the specific `config` struct. It takes the generic `globalConfig.ResourceConfig` and
+// decodes it into the `config` struct and validates the values.
+func newConfig(rc *globalConfig.ResourceConfig) (*config, error) {
+	var c metaConfig
+
+	err := mapstructure.Decode(rc, &c)
+	if err != nil {
+		slog.Error("unable to decode config into HashiCorp Vault source config", "error", err)
+		return nil, err
+	}
+
+	validate = validator.New()
+	if err := validate.Struct(c); err != nil {
+		return nil, err
+	}
+
+	return &c.hashicorpvault, nil
+}
+
+// LogValue customizes how the `config` struct will be printed in the logs.
+func (c *config) LogValue() slog.Value {
+	return slog.GroupValue(slog.String("address", c.Address), slog.String("token", "***"))
+}
+
+func (c *config) GetHashiCorpVaultConfig() *vault.Config {
+	config := vault.Config{
+		Address: c.Address,
+	}
+
+	return &config
+}

internal/source/hashicorpvault/reader.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 cluetec GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hashicorpvault
+
+import (
+	globalConfig "github.com/cluetec/lifeboat/internal/config"
+	vault "github.com/hashicorp/vault/api"
+	"io"
+	"log/slog"
+)
+
+const snapshotPath = "/sys/storage/raft/snapshot"
+
+// Reader implements the `io.ReaderClose` interface in order to read the backup from HashiCorp Vault.
+type Reader struct {
+	client *vault.Client
+	reader io.Reader
+}
+
+// NewReader initializes a new `Reader` struct which is implementing the `io.ReaderClose` interface.
+func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) {
+	c, err := newConfig(rc)
+	if err != nil {
+		slog.Error("error while initializing source config", "sourceType", Type, "error", err)
+		return nil, err
+	}
+
+	slog.Debug("source config loaded", "sourceType", Type, "config", c)
+
+	client, err := vault.NewClient(c.GetHashiCorpVaultConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	client.SetToken(c.Token)
+	return &Reader{client: client}, nil
+}
+
+func (r *Reader) Read(b []byte) (int, error) {
+	slog.Debug("read got called", "sourceType", Type)
+
+	if r.reader == nil {
+		resp, err := r.client.Logical().ReadRaw(snapshotPath)
+		if err != nil {
+			slog.Error("failed to called backup endpoint", "error", err)
+			return 0, err
+		}
+
+		r.reader = resp.Body
+	}
+
+	return r.reader.Read(b)
+}
+
+func (r *Reader) Close() error {
+	slog.Debug("closing reader", "sourceType", Type)
+	if closer, ok := r.reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}

internal/source/source.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"github.com/cluetec/lifeboat/internal/config"
 	"github.com/cluetec/lifeboat/internal/source/filesystem"
+	"github.com/cluetec/lifeboat/internal/source/hashicorpvault"
 	"io"
 	"log/slog"
 )
@@ -32,9 +33,11 @@ func New(c config.SourceConfig) (*Source, error) {
 	s := Source{}
 	var err error
 
-	switch {
-	case c.Type == filesystem.Type:
+	switch c.Type {
+	case filesystem.Type:
 		s.Reader, err = filesystem.NewReader(&c.ResourceConfig)
+	case hashicorpvault.Type:
+		s.Reader, err = hashicorpvault.NewReader(&c.ResourceConfig)
 	}
 	if err != nil {
 		slog.Error("error while initializing reader interface for source system", "sourceType", c.Type, "error", err)

samples/HashiCorp-Vault/.gitignore

@@ -0,0 +1,4 @@
+.data/
+backup-destination/*.snap
+vault-token.txt
+vault-unseal-keys.txt