From fe27b8cc1c2dc73487c0a416b062683f2c1d3bcc Mon Sep 17 00:00:00 2001 From: "Florian Rusch (cluetec GmbH)" Date: Mon, 13 Nov 2023 07:06:03 +0100 Subject: [PATCH 1/9] Implement cli flag for pointing to specific config file --- cmd/backup.go | 13 ++++++++++++- internal/config/config.go | 4 ++-- internal/config/viper.go | 11 ++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 191ed63..f3a08d6 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -26,13 +26,15 @@ import ( "log/slog" ) +var cfgFilePath string + // backupCmd represents the backup command var backupCmd = &cobra.Command{ Use: "backup", Short: "Execute the backup.", Long: "Execute the backup. Used config can be overridden by providing arguments.", RunE: func(cmd *cobra.Command, args []string) error { - c, err := config.New() + c, err := config.New(cfgFilePath) if err != nil { slog.Error("error while initializing config", "error", err) return err @@ -79,3 +81,12 @@ var backupCmd = &cobra.Command{ return nil }, } + +func init() { + rootCmd.PersistentFlags().StringVar( + &cfgFilePath, + "config", + "", + "path to config file (default: ./config.yaml)", + ) +} diff --git a/internal/config/config.go b/internal/config/config.go index bc2c5b8..2e6fd9c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -63,8 +63,8 @@ func (c *Config) DebugEnabled() bool { return strings.ToLower(c.LogLevel) == "debug" } -func New() (*Config, error) { - if err := initViper(); err != nil { +func New(cfgFilePath string) (*Config, error) { + if err := initViper(cfgFilePath); err != nil { slog.Error("error while initializing viper", "error", err) return nil, err } diff --git a/internal/config/viper.go b/internal/config/viper.go index 59bd377..015dd26 100644 --- a/internal/config/viper.go +++ b/internal/config/viper.go @@ -22,11 +22,16 @@ import ( "strings" ) -func initViper() error { +func initViper(cfgFilePath string) error { viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - viper.AddConfigPath(".") - viper.SetConfigFile("./config.yaml") + + if cfgFilePath == "" { + viper.AddConfigPath(".") + viper.SetConfigFile("./config.yaml") + } else { + viper.SetConfigFile(cfgFilePath) + } if err := viper.ReadInConfig(); err != nil { slog.Error("error while reading in the configs: %w", err) From 3ab185ecbed2e758d85799a2755bbedb3657c05d Mon Sep 17 00:00:00 2001 From: "Florian Rusch (cluetec GmbH)" Date: Fri, 24 Nov 2023 10:30:50 +0100 Subject: [PATCH 2/9] feat: add source system support for hashicorp vault --- config.yaml | 18 ++- go.mod | 21 ++++ go.sum | 74 ++++++++++++ internal/source/hashicorpvault/config.go | 70 +++++++++++ internal/source/hashicorpvault/reader.go | 75 ++++++++++++ internal/source/source.go | 7 +- samples/HashiCorp-Vault/.gitignore | 4 + samples/HashiCorp-Vault/README.md | 82 +++++++++++++ samples/HashiCorp-Vault/backup-config.yaml | 9 ++ .../backup-destination/.gitkeep | 0 samples/HashiCorp-Vault/docker-compose.yaml | 25 ++++ .../init-and-fill-vault-with-data.sh | 109 ++++++++++++++++++ samples/HashiCorp-Vault/vault.hcl | 15 +++ 13 files changed, 501 insertions(+), 8 deletions(-) create mode 100644 internal/source/hashicorpvault/config.go create mode 100644 internal/source/hashicorpvault/reader.go create mode 100644 samples/HashiCorp-Vault/.gitignore create mode 100644 samples/HashiCorp-Vault/README.md create mode 100644 samples/HashiCorp-Vault/backup-config.yaml create mode 100644 samples/HashiCorp-Vault/backup-destination/.gitkeep create mode 100644 samples/HashiCorp-Vault/docker-compose.yaml create mode 100755 samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh create mode 100644 samples/HashiCorp-Vault/vault.hcl diff --git a/config.yaml b/config.yaml index c7b23da..a22e50b 100644 --- a/config.yaml +++ b/config.yaml @@ -1,9 +1,15 @@ +# In order to enable environmental substitution, it is necessary to include all potential configurations in this +# configuration file. Otherwise, the environment variables will not be automatically added to the ResourceConfig +# (`mapstructure:",remain"`). + +loglevel: info + source: - type: filesystem - path: /tmp/source.txt + type: + token: + address: + path: destination: - type: filesystem - path: /tmp/destination.txt - -logLevel: debug + type: + path: diff --git a/go.mod b/go.mod index 693eea5..4fc9b4a 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,35 @@ module github.com/cluetec/lifeboat go 1.21 require ( + github.com/go-playground/validator/v10 v10.16.0 + github.com/hashicorp/vault/api v1.10.0 github.com/mitchellh/mapstructure v1.5.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.17.0 ) require ( + github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.6.6 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -23,9 +41,12 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.13.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 65146c4..598457f 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -57,13 +61,30 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= +github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -120,10 +141,35 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/vault/api v1.10.0 h1:/US7sIjWN6Imp4o/Rj1Ce2Nr5bki/AXi9vAW3p2tOJQ= +github.com/hashicorp/vault/api v1.10.0/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -139,8 +185,21 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -150,11 +209,15 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -174,12 +237,15 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -201,10 +267,13 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -272,6 +341,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -291,6 +362,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -343,6 +415,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/internal/source/hashicorpvault/config.go b/internal/source/hashicorpvault/config.go new file mode 100644 index 0000000..116c041 --- /dev/null +++ b/internal/source/hashicorpvault/config.go @@ -0,0 +1,70 @@ +/* + * Copyright 2023 cluetec GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hashicorpvault + +import ( + globalConfig "github.com/cluetec/lifeboat/internal/config" + "github.com/go-playground/validator/v10" + vault "github.com/hashicorp/vault/api" + "github.com/mitchellh/mapstructure" + "log/slog" +) + +const Type = "hashicorpvault" + +type config struct { + Address string `validate:"http_url,required"` + Token string `validate:"required"` +} + +var validate *validator.Validate + +// newConfig provides the specific `config` struct. Therefor it takes the generic `globalConfig.ResourceConfig` and +// decodes it into the `config` struct and validates the values. +func newConfig(rc *globalConfig.ResourceConfig) (*config, error) { + var c config + + err := mapstructure.Decode(rc, &c) + if err != nil { + slog.Error("unable to decode config into HashiCorp Vault source config", "error", err) + return nil, err + } + + validate = validator.New() + if err := validate.Struct(c); err != nil { + return nil, err + } + + return &c, nil +} + +// LogValue customizes how the `config` struct will be printed in the logs. +func (c *config) LogValue() slog.Value { + return slog.GroupValue(slog.String("address", c.Address), slog.String("token", "***")) +} + +// GetHashiCorpVaultConfig was implement in regard to the +// `vault.DefaultConfig()` method. While the implementation the +// `config.ReadEnvironment` was left of, to avoid the usage of additional +// environment variables like `VAULT_ADDRESS`. +func (c *config) GetHashiCorpVaultConfig() *vault.Config { + config := vault.Config{ + Address: c.Address, + } + + return &config +} diff --git a/internal/source/hashicorpvault/reader.go b/internal/source/hashicorpvault/reader.go new file mode 100644 index 0000000..5619baa --- /dev/null +++ b/internal/source/hashicorpvault/reader.go @@ -0,0 +1,75 @@ +/* + * Copyright 2023 cluetec GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hashicorpvault + +import ( + globalConfig "github.com/cluetec/lifeboat/internal/config" + vault "github.com/hashicorp/vault/api" + "io" + "log/slog" +) + +const snapshotPath = "/sys/storage/raft/snapshot" + +// Reader implements the `io.ReaderClose` interface for read the backup from HashiCorp Vault. +type Reader struct { + client *vault.Client + reader io.Reader +} + +// NewReader initializes a new `Reader` struct which is implementing the `io.ReaderClose` interface. +func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) { + c, err := newConfig(rc) + if err != nil { + slog.Error("error while initializing source config", "sourceType", "hashicorpvault", "error", err) + return nil, err + } + + slog.Debug("source config loaded", "sourceType", Type, "config", rc) + + client, err := vault.NewClient(c.GetHashiCorpVaultConfig()) + if err != nil { + return nil, err + } + + client.SetToken(c.Token) + return &Reader{client: client}, nil +} + +func (r *Reader) Read(b []byte) (int, error) { + slog.Debug("hashicorp vault source read got called") + + if r.reader == nil { + resp, err := r.client.Logical().ReadRaw(snapshotPath) + if err != nil { + slog.Error("failed to called backup endpoint", "error", err) + return 0, err + } + + r.reader = resp.Body + } + + return r.reader.Read(b) +} + +func (r *Reader) Close() error { + slog.Debug("closing HashiCorp Vault reader") + if closer, ok := r.reader.(io.Closer); ok { + return closer.Close() + } + return nil +} diff --git a/internal/source/source.go b/internal/source/source.go index 318050d..ea10e25 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -20,6 +20,7 @@ import ( "errors" "github.com/cluetec/lifeboat/internal/config" "github.com/cluetec/lifeboat/internal/source/filesystem" + "github.com/cluetec/lifeboat/internal/source/hashicorpvault" "io" "log/slog" ) @@ -32,9 +33,11 @@ func New(c config.SourceConfig) (*Source, error) { s := Source{} var err error - switch { - case c.Type == filesystem.Type: + switch c.Type { + case filesystem.Type: s.Reader, err = filesystem.NewReader(&c.ResourceConfig) + case hashicorpvault.Type: + s.Reader, err = hashicorpvault.NewReader(&c.ResourceConfig) } if err != nil { slog.Error("error while initializing reader interface for source system", "sourceType", c.Type, "error", err) diff --git a/samples/HashiCorp-Vault/.gitignore b/samples/HashiCorp-Vault/.gitignore new file mode 100644 index 0000000..737ead8 --- /dev/null +++ b/samples/HashiCorp-Vault/.gitignore @@ -0,0 +1,4 @@ +.data/ +backup-destination/*.snap +vault-token.txt +vault-unseal-keys.txt diff --git a/samples/HashiCorp-Vault/README.md b/samples/HashiCorp-Vault/README.md new file mode 100644 index 0000000..de16aec --- /dev/null +++ b/samples/HashiCorp-Vault/README.md @@ -0,0 +1,82 @@ +# Sample: Backup HashiCorp Vault + +In this example we will show you how to backup an HashiCorp Vault instance. + +## Requirements + +The Vault instance needs to use [raft](https://developer.hashicorp.com/vault/docs/configuration/storage/raft) as the +underlying storage engine. + +## Policy + +In this sample we are using for simplicity reasons the "root token" to authorize us. In a real world scenario you +would use a separate took or any other +[authentication method supported by vault](https://developer.hashicorp.com/vault/docs/auth). + +**What's important here**: Normally your identity shouldn't have root permissions! The only permission you need for +creating the backup/snapshot is the following +([vault policy](https://developer.hashicorp.com/vault/docs/concepts/policies)): + +```hcl +path "/sys/storage/raft/snapshot" { + capabilities = ["read"] +} +``` + +## Run + +### 1. Start the vault instance + +The HashiCorp Vault setup is based on a docker compose setup. + +```shell +# -d is starting the container in the background +$ docker-compose up -d +``` + +As we can't simply use the dev mode of Vault, we need to initialize and unseal it first. For this purpose the +`docker-compose.yaml` contains next to the vault container an additional one which is called `vault-init`. This +container contains the bash script `./init-and-fill-vault.sh` which will do all the necessary steps. + +On default, we are storing 1000 secrets with a length of 2000 random chars into vault. As this takes some seconds, you +can verify with this command, if the container has successfully executed the script or not. As a hint, it could take +something around 1 minute until the init script finishes. + +- Successful: Status of vault-init container == `Exited (0)` +- Not successful: Status of vault-init container == `Exited (1)` + +```shell +$ docker-compose ps --all +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +hashicorp-vault-vault-1 hashicorp/vault:1.15 "vault server -confi…" vault 59 seconds ago Up 58 seconds 0.0.0.0:8200->8200/tcp +hashicorp-vault-vault-init-1 hashicorp-vault-vault-init "bash /init.sh" vault-init 59 seconds ago Exited (0) 5 seconds ago +``` + +### 3. Run lifeboat to create the backup + +As the root token will be randomly generated everytime you are starting a new vault instance, we are storing it in the +file `./vault-token.txt` so that we can use it in lifeboat to successfully authenticate while doing the backup. +Therefor we need to parse the content of this file into an environment variable which will be then used by +lifeboat. + +The following command will trigger a backup and will store it in the `./backup-destination` folder: + +```shell +$ SOURCE_TOKEN=$(cat ./vault-token.txt) lb backup --config ./backup-config.yaml +``` + +## Clean up after run + +To clean up everything afterwards, we just need to execute the following commands: + +```shell +#$ docker-compose down +#$ rm -rf .data +$ rm -rf backup-destination/vault-backup.snap +``` + + +## Restore + +An official guide how to restore a backup/snapshot can be found here: + diff --git a/samples/HashiCorp-Vault/backup-config.yaml b/samples/HashiCorp-Vault/backup-config.yaml new file mode 100644 index 0000000..5b9324c --- /dev/null +++ b/samples/HashiCorp-Vault/backup-config.yaml @@ -0,0 +1,9 @@ +source: + type: hashicorpvault + # the token will be provided via env variable + token: + address: http://127.0.0.1:8200 + +destination: + type: filesystem + path: ./backup-destination/vault-backup.snap diff --git a/samples/HashiCorp-Vault/backup-destination/.gitkeep b/samples/HashiCorp-Vault/backup-destination/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/samples/HashiCorp-Vault/docker-compose.yaml b/samples/HashiCorp-Vault/docker-compose.yaml new file mode 100644 index 0000000..3c06eaf --- /dev/null +++ b/samples/HashiCorp-Vault/docker-compose.yaml @@ -0,0 +1,25 @@ +services: + vault: + image: hashicorp/vault:1.15 + ports: + - "8200:8200" + volumes: + - ./.data:/vault/data:rw + - ./vault.hcl:/vault/vault.hcl:rw + cap_add: + - IPC_LOCK + entrypoint: vault server -config=/vault/vault.hcl + +# vault-init: +# build: +# context: . +# dockerfile_inline: | +# FROM hashicorp/vault:1.15 +# RUN apk update && \ +# apk add bash +# no_cache: true +# volumes: +# - ./init-and-fill-vault-with-data.sh:/init.sh:ro +# - ./vault-token.txt:/vault-token.txt:rw +# - ./vault-unseal-keys.txt:/vault-unseal-keys.txt:rw +# entrypoint: bash /init.sh diff --git a/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh new file mode 100755 index 0000000..4cdb052 --- /dev/null +++ b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh @@ -0,0 +1,109 @@ +# +# Copyright 2023 cluetec GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#!/bin/bash + +set -o errexit +set -o errtrace +set -o pipefail +set -o nounset + +export VAULT_ADDR="http://vault:8200" + +# Check if vault is up and running +attempt_counter=0 +max_attempts=3 +sleep_seconds=5 +until $(vault status > /dev/null 2>&1); do + if [ $? -eq 2 ]; then + echo "Vault is up and sealed" + break; + fi + + if [ ${attempt_counter} -eq ${max_attempts} ]; then + echo "" + echo "ERROR: The Vault was not started in time. Maximum connection attempts of ${max_attempts} have been reached and $((sleep_seconds*max_attempts)) seconds have been waited. Please have a look at the Vault instance to see why it did not start in time." + exit 1 + fi + + echo "Waiting until vault is up..." + sleep ${sleep_seconds} + attempt_counter=$(($attempt_counter+1)) +done + +# Initialize vault +initOutput=$(vault operator init) +#initOutput="Unseal Key 1: xxxx +#Unseal Key 2: xxxx +#Unseal Key 3: xxxx +#Unseal Key 4: xxxx +#Unseal Key 5: xxxx +# +#Initial Root Token: xxxxx +# +#Vault initialized with 5 key shares and a key threshold of 3. Please securely +#distribute the key shares printed above. When the Vault is re-sealed, +#restarted, or stopped, you must supply at least 3 of these keys to unseal it +#before it can start servicing requests. +# +#Vault does not store the generated root key. Without at least 3 keys to +#reconstruct the root key, Vault will remain permanently sealed! +# +#It is possible to generate new unseal keys, provided you have a quorum of +#existing unseal keys shares. See \"vault operator rekey\" for more information.: No such file or directory" + +# Get unseal keys +unsealKeys=$(echo "$initOutput" | grep "^Unseal Key ") + +# Get root token +while IFS= read -r SINGLELINE +do + re="Initial Root Token: " + if [[ "${SINGLELINE}" =~ $re ]]; then + rootToken=$(echo "${SINGLELINE}" | sed "s/${re}//") + fi +done << EOF +$initOutput +EOF + +# Write unseal keys und token into separate files +echo "${unsealKeys}" > vault-unseal-keys.txt +echo "${rootToken}" > vault-token.txt + +# Unseal vault +printf "\nFirst unseal command:\n" +vault operator unseal $(echo "${unsealKeys}" | sed -n 1p | sed 's/Unseal Key 1: //') +printf "\nSecond unseal command:\n" +vault operator unseal $(echo "${unsealKeys}" | sed -n 2p | sed 's/Unseal Key 2: //') +printf "\nThird unseal command:\n" +vault operator unseal $(echo "${unsealKeys}" | sed -n 3p | sed 's/Unseal Key 3: //') + +# Login to vault +echo "$rootToken" | vault login - + +# Enable secret engine +vault secrets enable -version=2 -path="secret" kv + +# Put secrets into vault +amountOfSecrets=1000 +secretLength=2000 +for i in `seq 2 $amountOfSecrets`; do + printf "\nPut secret number ${i} into vault:\n" + superSecureSecret=$(sed "s/[^a-zA-Z0-9]//g" <<< $(cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9!@#$%*()-+' | fold -w ${secretLength} | head -n 1)) + echo "${superSecureSecret}" | vault kv put secret/${i} content=- +done + +printf "\nSuccessful initialized vault and put ${amountOfSecrets} with a length of ${secretLength} random chars into vault\n" diff --git a/samples/HashiCorp-Vault/vault.hcl b/samples/HashiCorp-Vault/vault.hcl new file mode 100644 index 0000000..9f0b1d4 --- /dev/null +++ b/samples/HashiCorp-Vault/vault.hcl @@ -0,0 +1,15 @@ +ui = true + +api_addr = "http://127.0.0.1:8200" +cluster_addr = "https://127.0.0.1:8201" + +listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" +} + +disable_mlock = true +storage "raft" { + path = "/vault/data" +} From 9badff29675b3d7f716ac4c2066fe40398aeac04 Mon Sep 17 00:00:00 2001 From: "Florian Rusch (cluetec GmbH)" Date: Fri, 24 Nov 2023 10:38:01 +0100 Subject: [PATCH 3/9] Fix some stylings --- .editorconfig | 3 +++ samples/HashiCorp-Vault/README.md | 9 +++---- samples/HashiCorp-Vault/docker-compose.yaml | 26 +++++++++---------- .../init-and-fill-vault-with-data.sh | 20 +++++++------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.editorconfig b/.editorconfig index 4c7771c..36fcc3f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,6 @@ trim_trailing_whitespace = true [{*.go,Makefile,go.mod,go.sum}] indent_style = tab + +[*.sh] +indent_size = 4 diff --git a/samples/HashiCorp-Vault/README.md b/samples/HashiCorp-Vault/README.md index de16aec..0915eb2 100644 --- a/samples/HashiCorp-Vault/README.md +++ b/samples/HashiCorp-Vault/README.md @@ -1,6 +1,6 @@ # Sample: Backup HashiCorp Vault -In this example we will show you how to backup an HashiCorp Vault instance. +In this example we will show you how to back up an HashiCorp Vault instance. ## Requirements @@ -47,9 +47,9 @@ something around 1 minute until the init script finishes. ```shell $ docker-compose ps --all -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -hashicorp-vault-vault-1 hashicorp/vault:1.15 "vault server -confi…" vault 59 seconds ago Up 58 seconds 0.0.0.0:8200->8200/tcp -hashicorp-vault-vault-init-1 hashicorp-vault-vault-init "bash /init.sh" vault-init 59 seconds ago Exited (0) 5 seconds ago +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +hashicorp-vault-vault-1 hashicorp/vault:1.15 "vault server -confi…" vault 59 seconds ago Up 58 seconds 0.0.0.0:8200->8200/tcp +hashicorp-vault-vault-init-1 hashicorp-vault-vault-init "bash /init.sh" vault-init 59 seconds ago Exited (0) 5 seconds ago ``` ### 3. Run lifeboat to create the backup @@ -75,7 +75,6 @@ To clean up everything afterwards, we just need to execute the following command $ rm -rf backup-destination/vault-backup.snap ``` - ## Restore An official guide how to restore a backup/snapshot can be found here: diff --git a/samples/HashiCorp-Vault/docker-compose.yaml b/samples/HashiCorp-Vault/docker-compose.yaml index 3c06eaf..1d06af5 100644 --- a/samples/HashiCorp-Vault/docker-compose.yaml +++ b/samples/HashiCorp-Vault/docker-compose.yaml @@ -10,16 +10,16 @@ services: - IPC_LOCK entrypoint: vault server -config=/vault/vault.hcl -# vault-init: -# build: -# context: . -# dockerfile_inline: | -# FROM hashicorp/vault:1.15 -# RUN apk update && \ -# apk add bash -# no_cache: true -# volumes: -# - ./init-and-fill-vault-with-data.sh:/init.sh:ro -# - ./vault-token.txt:/vault-token.txt:rw -# - ./vault-unseal-keys.txt:/vault-unseal-keys.txt:rw -# entrypoint: bash /init.sh + vault-init: + build: + context: . + dockerfile_inline: | + FROM hashicorp/vault:1.15 + RUN apk update && \ + apk add bash + no_cache: true + volumes: + - ./init-and-fill-vault-with-data.sh:/init.sh:ro + - ./vault-token.txt:/vault-token.txt:rw + - ./vault-unseal-keys.txt:/vault-unseal-keys.txt:rw + entrypoint: bash /init.sh diff --git a/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh index 4cdb052..ee8d60a 100755 --- a/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh +++ b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh @@ -34,9 +34,9 @@ until $(vault status > /dev/null 2>&1); do fi if [ ${attempt_counter} -eq ${max_attempts} ]; then - echo "" - echo "ERROR: The Vault was not started in time. Maximum connection attempts of ${max_attempts} have been reached and $((sleep_seconds*max_attempts)) seconds have been waited. Please have a look at the Vault instance to see why it did not start in time." - exit 1 + echo "" + echo "ERROR: The Vault was not started in time. Maximum connection attempts of ${max_attempts} have been reached and $((sleep_seconds*max_attempts)) seconds have been waited. Please have a look at the Vault instance to see why it did not start in time." + exit 1 fi echo "Waiting until vault is up..." @@ -71,10 +71,10 @@ unsealKeys=$(echo "$initOutput" | grep "^Unseal Key ") # Get root token while IFS= read -r SINGLELINE do - re="Initial Root Token: " - if [[ "${SINGLELINE}" =~ $re ]]; then - rootToken=$(echo "${SINGLELINE}" | sed "s/${re}//") - fi + re="Initial Root Token: " + if [[ "${SINGLELINE}" =~ $re ]]; then + rootToken=$(echo "${SINGLELINE}" | sed "s/${re}//") + fi done << EOF $initOutput EOF @@ -101,9 +101,9 @@ vault secrets enable -version=2 -path="secret" kv amountOfSecrets=1000 secretLength=2000 for i in `seq 2 $amountOfSecrets`; do - printf "\nPut secret number ${i} into vault:\n" - superSecureSecret=$(sed "s/[^a-zA-Z0-9]//g" <<< $(cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9!@#$%*()-+' | fold -w ${secretLength} | head -n 1)) - echo "${superSecureSecret}" | vault kv put secret/${i} content=- + printf "\nPut secret number ${i} into vault:\n" + superSecureSecret=$(sed "s/[^a-zA-Z0-9]//g" <<< $(cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9!@#$%*()-+' | fold -w ${secretLength} | head -n 1)) + echo "${superSecureSecret}" | vault kv put secret/${i} content=- done printf "\nSuccessful initialized vault and put ${amountOfSecrets} with a length of ${secretLength} random chars into vault\n" From 559b1fc7803af9a44b39a0b2337584302a49b2af Mon Sep 17 00:00:00 2001 From: "Florian Rusch (cluetec GmbH)" Date: Fri, 24 Nov 2023 10:44:26 +0100 Subject: [PATCH 4/9] Uncomment clean up stuff --- samples/HashiCorp-Vault/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/HashiCorp-Vault/README.md b/samples/HashiCorp-Vault/README.md index 0915eb2..fdcd9da 100644 --- a/samples/HashiCorp-Vault/README.md +++ b/samples/HashiCorp-Vault/README.md @@ -70,8 +70,8 @@ $ SOURCE_TOKEN=$(cat ./vault-token.txt) lb backup --config ./backup-config.yaml To clean up everything afterwards, we just need to execute the following commands: ```shell -#$ docker-compose down -#$ rm -rf .data +$ docker-compose down +$ rm -rf .data $ rm -rf backup-destination/vault-backup.snap ``` From 48ec82f234887aed7fc0d7c5ed9e9f8a5b7b6d2b Mon Sep 17 00:00:00 2001 From: Florian Rusch Date: Thu, 4 Jan 2024 13:28:18 +0100 Subject: [PATCH 5/9] Move config flag from rootCmd to backupCmd --- cmd/backup.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/backup.go b/cmd/backup.go index f3a08d6..bec73b2 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -83,9 +83,10 @@ var backupCmd = &cobra.Command{ } func init() { - rootCmd.PersistentFlags().StringVar( + backupCmd.PersistentFlags().StringVarP( &cfgFilePath, "config", + "c", "", "path to config file (default: ./config.yaml)", ) From 091d7c411861c04abb10c8213e60198ed0faa3fb Mon Sep 17 00:00:00 2001 From: Florian Rusch Date: Fri, 5 Jan 2024 09:13:10 +0100 Subject: [PATCH 6/9] Fix wrong type in filesystem config --- internal/source/filesystem/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/source/filesystem/config.go b/internal/source/filesystem/config.go index 2a1d6e0..bb5a615 100644 --- a/internal/source/filesystem/config.go +++ b/internal/source/filesystem/config.go @@ -25,7 +25,7 @@ import ( const Type = "filesystem" type metaConfig struct { - Filesystem Config + Filesystem config } type config struct { From a61b74a91c347541f5f86ac309750aa8e7810ade Mon Sep 17 00:00:00 2001 From: Florian Rusch Date: Fri, 5 Jan 2024 09:13:47 +0100 Subject: [PATCH 7/9] Add metaConfig layer and fix types and wordings --- internal/source/hashicorpvault/config.go | 14 +++++++------- internal/source/hashicorpvault/reader.go | 2 +- samples/HashiCorp-Vault/README.md | 8 ++++---- .../init-and-fill-vault-with-data.sh | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/source/hashicorpvault/config.go b/internal/source/hashicorpvault/config.go index 116c041..ed8594d 100644 --- a/internal/source/hashicorpvault/config.go +++ b/internal/source/hashicorpvault/config.go @@ -26,6 +26,10 @@ import ( const Type = "hashicorpvault" +type metaConfig struct { + hashicorpvault config +} + type config struct { Address string `validate:"http_url,required"` Token string `validate:"required"` @@ -33,10 +37,10 @@ type config struct { var validate *validator.Validate -// newConfig provides the specific `config` struct. Therefor it takes the generic `globalConfig.ResourceConfig` and +// newConfig provides the specific `config` struct. It takes the generic `globalConfig.ResourceConfig` and // decodes it into the `config` struct and validates the values. func newConfig(rc *globalConfig.ResourceConfig) (*config, error) { - var c config + var c metaConfig err := mapstructure.Decode(rc, &c) if err != nil { @@ -49,7 +53,7 @@ func newConfig(rc *globalConfig.ResourceConfig) (*config, error) { return nil, err } - return &c, nil + return &c.hashicorpvault, nil } // LogValue customizes how the `config` struct will be printed in the logs. @@ -57,10 +61,6 @@ func (c *config) LogValue() slog.Value { return slog.GroupValue(slog.String("address", c.Address), slog.String("token", "***")) } -// GetHashiCorpVaultConfig was implement in regard to the -// `vault.DefaultConfig()` method. While the implementation the -// `config.ReadEnvironment` was left of, to avoid the usage of additional -// environment variables like `VAULT_ADDRESS`. func (c *config) GetHashiCorpVaultConfig() *vault.Config { config := vault.Config{ Address: c.Address, diff --git a/internal/source/hashicorpvault/reader.go b/internal/source/hashicorpvault/reader.go index 5619baa..e30656d 100644 --- a/internal/source/hashicorpvault/reader.go +++ b/internal/source/hashicorpvault/reader.go @@ -25,7 +25,7 @@ import ( const snapshotPath = "/sys/storage/raft/snapshot" -// Reader implements the `io.ReaderClose` interface for read the backup from HashiCorp Vault. +// Reader implements the `io.ReaderClose` interface in order to read the backup from HashiCorp Vault. type Reader struct { client *vault.Client reader io.Reader diff --git a/samples/HashiCorp-Vault/README.md b/samples/HashiCorp-Vault/README.md index fdcd9da..35f8d6f 100644 --- a/samples/HashiCorp-Vault/README.md +++ b/samples/HashiCorp-Vault/README.md @@ -1,6 +1,6 @@ # Sample: Backup HashiCorp Vault -In this example we will show you how to back up an HashiCorp Vault instance. +In this example we will show you how to backup an HashiCorp Vault instance. ## Requirements @@ -10,7 +10,7 @@ underlying storage engine. ## Policy In this sample we are using for simplicity reasons the "root token" to authorize us. In a real world scenario you -would use a separate took or any other +would use a separate token or any other [authentication method supported by vault](https://developer.hashicorp.com/vault/docs/auth). **What's important here**: Normally your identity shouldn't have root permissions! The only permission you need for @@ -35,8 +35,8 @@ $ docker-compose up -d ``` As we can't simply use the dev mode of Vault, we need to initialize and unseal it first. For this purpose the -`docker-compose.yaml` contains next to the vault container an additional one which is called `vault-init`. This -container contains the bash script `./init-and-fill-vault.sh` which will do all the necessary steps. +`docker-compose.yaml` contains an additional container which is called `vault-init`. This container contains the bash +script `./init-and-fill-vault.sh` which will do all the necessary steps. On default, we are storing 1000 secrets with a length of 2000 random chars into vault. As this takes some seconds, you can verify with this command, if the container has successfully executed the script or not. As a hint, it could take diff --git a/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh index ee8d60a..2759596 100755 --- a/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh +++ b/samples/HashiCorp-Vault/init-and-fill-vault-with-data.sh @@ -79,7 +79,7 @@ done << EOF $initOutput EOF -# Write unseal keys und token into separate files +# Write unseal keys and token into separate files echo "${unsealKeys}" > vault-unseal-keys.txt echo "${rootToken}" > vault-token.txt From 03c3cc88f47412523ad62110ab3ca5c9646c92b8 Mon Sep 17 00:00:00 2001 From: Florian Rusch Date: Fri, 5 Jan 2024 09:14:21 +0100 Subject: [PATCH 8/9] Cleanup config --- config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config.yaml b/config.yaml index 00db7dd..0b6eb0d 100644 --- a/config.yaml +++ b/config.yaml @@ -19,5 +19,3 @@ destination: filesystem: path: /tmp/destination.txt - -logLevel: debug From b16a37b7ea7ff81334b1a4c9fe56bef0968a85c8 Mon Sep 17 00:00:00 2001 From: Florian Rusch Date: Fri, 5 Jan 2024 09:56:12 +0100 Subject: [PATCH 9/9] Streamline logs and error messages --- internal/destination/filesystem/writer.go | 9 +++++---- internal/source/filesystem/reader.go | 8 ++++---- internal/source/hashicorpvault/reader.go | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/internal/destination/filesystem/writer.go b/internal/destination/filesystem/writer.go index 0cfb4d7..c16e394 100644 --- a/internal/destination/filesystem/writer.go +++ b/internal/destination/filesystem/writer.go @@ -30,18 +30,18 @@ type Writer struct { func NewWriter(rc *globalConfig.ResourceConfig) (*Writer, error) { c, err := newConfig(rc) if err != nil { - slog.Error("error while initializing filesystem destination config", "error", err) + slog.Error("error while initializing destination config", "destinationType", Type, "error", err) return nil, err } - slog.Debug("filesystem destination config loaded", "config", c) + slog.Debug("destination config loaded", "destinationType", Type, "config", c) // Check if destination file already exists _, err = os.Stat(c.Path) if err == nil { return nil, errors.New("destination file already exists") } else if !errors.Is(err, os.ErrNotExist) { - slog.Error("error while checking if destination file already exists", "error", err) + slog.Error("error while checking if destination file already exists", "destinationType", Type, "error", err) return nil, err } @@ -56,11 +56,12 @@ func NewWriter(rc *globalConfig.ResourceConfig) (*Writer, error) { func (w *Writer) Write(b []byte) (int, error) { slog.Debug("filesystem destination write got called") + slog.Debug("write got called", "destinationType", Type) return w.file.Write(b) } func (w *Writer) Close() error { - slog.Debug("closing filesystem writer") + slog.Debug("closing writer", "destinationType", Type) if w.file != nil { if err := w.file.Close(); err != nil { diff --git a/internal/source/filesystem/reader.go b/internal/source/filesystem/reader.go index 7020475..b3d7422 100644 --- a/internal/source/filesystem/reader.go +++ b/internal/source/filesystem/reader.go @@ -29,11 +29,11 @@ type Reader struct { func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) { c, err := newConfig(rc) if err != nil { - slog.Error("error while initializing filesystem source config", "error", err) + slog.Error("error while initializing source config", "sourceType", Type, "error", err) return nil, err } - slog.Debug("filesystem source config loaded", "config", rc) + slog.Debug("source config loaded", "sourceType", Type, "config", c) f, err := os.Open(c.Path) if err != nil { @@ -44,12 +44,12 @@ func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) { } func (r *Reader) Read(b []byte) (int, error) { - slog.Debug("filesystem source read got called") + slog.Debug("read got called", "sourceType", Type) return r.file.Read(b) } func (r *Reader) Close() error { - slog.Debug("closing filesystem reader") + slog.Debug("closing reader", "sourceType", Type) if r.file != nil { if err := r.file.Close(); err != nil { diff --git a/internal/source/hashicorpvault/reader.go b/internal/source/hashicorpvault/reader.go index e30656d..67d34b7 100644 --- a/internal/source/hashicorpvault/reader.go +++ b/internal/source/hashicorpvault/reader.go @@ -35,11 +35,11 @@ type Reader struct { func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) { c, err := newConfig(rc) if err != nil { - slog.Error("error while initializing source config", "sourceType", "hashicorpvault", "error", err) + slog.Error("error while initializing source config", "sourceType", Type, "error", err) return nil, err } - slog.Debug("source config loaded", "sourceType", Type, "config", rc) + slog.Debug("source config loaded", "sourceType", Type, "config", c) client, err := vault.NewClient(c.GetHashiCorpVaultConfig()) if err != nil { @@ -51,7 +51,7 @@ func NewReader(rc *globalConfig.ResourceConfig) (*Reader, error) { } func (r *Reader) Read(b []byte) (int, error) { - slog.Debug("hashicorp vault source read got called") + slog.Debug("read got called", "sourceType", Type) if r.reader == nil { resp, err := r.client.Logical().ReadRaw(snapshotPath) @@ -67,7 +67,7 @@ func (r *Reader) Read(b []byte) (int, error) { } func (r *Reader) Close() error { - slog.Debug("closing HashiCorp Vault reader") + slog.Debug("closing reader", "sourceType", Type) if closer, ok := r.reader.(io.Closer); ok { return closer.Close() }