feat: add source system support for hashicorp vault

config.yaml

@@ -1,9 +1,15 @@
+# In order to enable environmental substitution, it is necessary to include all potential configurations in this
+# configuration file. Otherwise, the environment variables will not be automatically added to the ResourceConfig
+# (`mapstructure:",remain"`).
+
+loglevel: info
+
 source:
-  type: filesystem
-  path: /tmp/source.txt
+  type:
+  token:
+  address:
+  path:
 
 destination:
-  type: filesystem
-  path: /tmp/destination.txt
-
-logLevel: debug
+  type:
+  path:

go.mod

@@ -3,17 +3,35 @@ module github.com/cluetec/lifeboat
 go 1.21
 
 require (
+	github.com/go-playground/validator/v10 v10.16.0
+	github.com/hashicorp/vault/api v1.10.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.17.0
 )
 
 require (
+	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.6 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -23,9 +41,12 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/source/hashicorpvault/config.go

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2023 cluetec GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hashicorpvault
+
+import (
+	globalConfig "github.com/cluetec/lifeboat/internal/config"
+	"github.com/go-playground/validator/v10"
+	vault "github.com/hashicorp/vault/api"
+	"github.com/mitchellh/mapstructure"
+	"log/slog"
+)
+
+const Type = "hashicorpvault"
+
+type config struct {
+	Address string `validate:"http_url,required"`
+	Token   string `validate:"required"`
+}
+
+var validate *validator.Validate
+
+// newConfig provides the specific `config` struct. Therefor it takes the generic `globalConfig.ResourceConfig` and
+// decodes it into the `config` struct and validates the values.
+func newConfig(rc *globalConfig.ResourceConfig) (*config, error) {
+	var c config
+
+	err := mapstructure.Decode(rc, &c)
+	if err != nil {
+		slog.Error("unable to decode config into HashiCorp Vault source config", "error", err)
+		return nil, err
+	}
+
+	validate = validator.New()
+	if err := validate.Struct(c); err != nil {
+		return nil, err
+	}
+
+	return &c, nil
+}
+
+// LogValue customizes how the `config` struct will be printed in the logs.
+func (c *config) LogValue() slog.Value {
+	return slog.GroupValue(slog.String("address", c.Address), slog.String("token", "***"))
+}
+
+// GetHashiCorpVaultConfig was implement in regard to the
+// `vault.DefaultConfig()` method. While the implementation the
+// `config.ReadEnvironment` was left of, to avoid the usage of additional
+// environment variables like `VAULT_ADDRESS`.
+func (c *config) GetHashiCorpVaultConfig() *vault.Config {
+	config := vault.Config{
+		Address: c.Address,
+	}
+
+	return &config
+}

internal/source/hashicorpvault/reader.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 cluetec GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hashicorpvault
+
+import (
+	globalConfig "github.com/cluetec/lifeboat/internal/config"
+	vault "github.com/hashicorp/vault/api"
+	"io"
+	"log/slog"
+)
+
+const snapshotPath = "/sys/storage/raft/snapshot"
+
+// Reader implements the `io.ReaderClose` interface for read the backup from HashiCorp Vault.
+type Reader struct {
+	client *vault.Client
+	reader io.Reader
+}
+
+// NewReader initializes a new `Reader` struct which is implementing the `io.ReaderClose` interface.
+func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) {
+	c, err := newConfig(rc)
+	if err != nil {
+		slog.Error("error while initializing source config", "sourceType", "hashicorpvault", "error", err)
+		return nil, err
+	}
+
+	slog.Debug("source config loaded", "sourceType", Type, "config", rc)
+
+	client, err := vault.NewClient(c.GetHashiCorpVaultConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	client.SetToken(c.Token)
+	return &Reader{client: client}, nil
+}
+
+func (r *Reader) Read(b []byte) (int, error) {
+	slog.Debug("hashicorp vault source read got called")
+
+	if r.reader == nil {
+		resp, err := r.client.Logical().ReadRaw(snapshotPath)
+		if err != nil {
+			slog.Error("failed to called backup endpoint", "error", err)
+			return 0, err
+		}
+
+		r.reader = resp.Body
+	}
+
+	return r.reader.Read(b)
+}
+
+func (r *Reader) Close() error {
+	slog.Debug("closing HashiCorp Vault reader")
+	if closer, ok := r.reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}

internal/source/source.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"github.com/cluetec/lifeboat/internal/config"
 	"github.com/cluetec/lifeboat/internal/source/filesystem"
+	"github.com/cluetec/lifeboat/internal/source/hashicorpvault"
 	"io"
 	"log/slog"
 )
@@ -32,9 +33,11 @@ func New(c config.SourceConfig) (*Source, error) {
 	s := Source{}
 	var err error
 
-	switch {
-	case c.Type == filesystem.Type:
+	switch c.Type {
+	case filesystem.Type:
 		s.Reader, err = filesystem.NewReader(&c.ResourceConfig)
+	case hashicorpvault.Type:
+		s.Reader, err = hashicorpvault.NewReader(&c.ResourceConfig)
 	}
 	if err != nil {
 		slog.Error("error while initializing reader interface for source system", "sourceType", c.Type, "error", err)

samples/HashiCorp-Vault/.gitignore

@@ -0,0 +1,4 @@
+.data/
+backup-destination/*.snap
+vault-token.txt
+vault-unseal-keys.txt

samples/HashiCorp-Vault/README.md

@@ -0,0 +1,82 @@
+# Sample: Backup HashiCorp Vault
+
+In this example we will show you how to backup an HashiCorp Vault instance.
+
+## Requirements
+
+The Vault instance needs to use [raft](https://developer.hashicorp.com/vault/docs/configuration/storage/raft) as the
+underlying storage engine.
+
+## Policy
+
+In this sample we are using for simplicity reasons the "root token" to authorize us. In a real world scenario you
+would use a separate took or any other
+[authentication method supported by vault](https://developer.hashicorp.com/vault/docs/auth).
+
+**What's important here**: Normally your identity shouldn't have root permissions! The only permission you need for
+creating the backup/snapshot is the following
+([vault policy](https://developer.hashicorp.com/vault/docs/concepts/policies)):
+
+```hcl
+path "/sys/storage/raft/snapshot" {
+  capabilities = ["read"]
+}
+```
+
+## Run
+
+### 1. Start the vault instance
+
+The HashiCorp Vault setup is based on a docker compose setup.
+
+```shell
+# -d is starting the container in the background
+$ docker-compose up -d
+```
+
+As we can't simply use the dev mode of Vault, we need to initialize and unseal it first. For this purpose the
+`docker-compose.yaml` contains next to the vault container an additional one which is called `vault-init`. This
+container contains the bash script `./init-and-fill-vault.sh` which will do all the necessary steps.
+
+On default, we are storing 1000 secrets with a length of 2000 random chars into vault. As this takes some seconds, you
+can verify with this command, if the container has successfully executed the script or not. As a hint, it could take
+something around 1 minute until the init script finishes.
+
+- Successful: Status of vault-init container == `Exited (0)`
+- Not successful: Status of vault-init container == `Exited (1)`
+
+```shell
+$ docker-compose ps --all
+NAME                                          IMAGE                        COMMAND                  SERVICE      CREATED          STATUS                     PORTS
+hashicorp-vault-vault-1                       hashicorp/vault:1.15         "vault server -confi…"   vault        59 seconds ago   Up 58 seconds              0.0.0.0:8200->8200/tcp
+hashicorp-vault-vault-init-1                  hashicorp-vault-vault-init   "bash /init.sh"          vault-init   59 seconds ago   Exited (0) 5 seconds ago
+```
+
+### 3. Run lifeboat to create the backup
+
+As the root token will be randomly generated everytime you are starting a new vault instance, we are storing it in the
+file `./vault-token.txt` so that we can use it in lifeboat to successfully authenticate while doing the backup.
+Therefor we need to parse the content of this file into an environment variable which will be then used by
+lifeboat.
+
+The following command will trigger a backup and will store it in the `./backup-destination` folder:
+
+```shell
+$ SOURCE_TOKEN=$(cat ./vault-token.txt) lb backup --config ./backup-config.yaml
+```
+
+## Clean up after run
+
+To clean up everything afterwards, we just need to execute the following commands:
+
+```shell
+#$ docker-compose down
+#$ rm -rf .data
+$ rm -rf backup-destination/vault-backup.snap
+```
+
+
+## Restore
+
+An official guide how to restore a backup/snapshot can be found here:
+<https://developer.hashicorp.com/vault/tutorials/standard-procedures/sop-restore>

samples/HashiCorp-Vault/backup-config.yaml

@@ -0,0 +1,9 @@
+source:
+  type: hashicorpvault
+  # the token will be provided via env variable
+  token:
+  address: http://127.0.0.1:8200
+
+destination:
+  type: filesystem
+  path: ./backup-destination/vault-backup.snap

samples/HashiCorp-Vault/docker-compose.yaml

@@ -0,0 +1,25 @@
+services:
+  vault:
+    image: hashicorp/vault:1.15
+    ports:
+      - "8200:8200"
+    volumes:
+      - ./.data:/vault/data:rw
+      - ./vault.hcl:/vault/vault.hcl:rw
+    cap_add:
+      - IPC_LOCK
+    entrypoint: vault server -config=/vault/vault.hcl
+
+#  vault-init:
+#    build:
+#      context: .
+#      dockerfile_inline: |
+#        FROM hashicorp/vault:1.15
+#        RUN apk update && \
+#            apk add bash
+#      no_cache: true
+#    volumes:
+#      - ./init-and-fill-vault-with-data.sh:/init.sh:ro
+#      - ./vault-token.txt:/vault-token.txt:rw
+#      - ./vault-unseal-keys.txt:/vault-unseal-keys.txt:rw
+#    entrypoint: bash /init.sh