diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..5e0fe9c --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,29 @@ +Running make & restore locally to test encryption & sending & receiving +----------------------------------------------------------------------- + +#### Server part + +1. In backup-repository repository you need to run `make k3d skaffold-deploy`, then you will have a working Backup Repository instance in local Kubernetes. + +2. Setup a tunnel for client connections + +```bash +kubectl port-forward svc/server-backup-repository-server -n backups 8080:8080 +``` + +#### Client part + +```bash +# export basic settings. Execute once in a console +export BM_COLLECTION_ID=iwa-ait +export BM_PASSPHRASE=riotkit +export BM_AUTH_TOKEN=$(curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://127.0.0.1:8080/api/stable/auth/login' | jq '.data.token' -r) +export BM_URL=http://127.0.0.1:8080 +``` + +#### Perform testing + +```bash +./.build/backup-maker make --cmd "tar -zcvf - ./" --key ./resources/test/gpg-key.asc +./.build/backup-maker restore --cmd "cat - > /tmp/restore.tar.gz" --passphrase riotkit --private-key ./resources/test/gpg-key.asc +``` diff --git a/README.md b/README.md index 055bc5a..4cf505f 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ export BM_PASSPHRASE="riotkit"; \ backup-maker make --url https://example.org \ -c "tar -zcvf - ./" \ --key build/test/backup.key \ - --recipient test@riotkit.org \ --log-level info ``` @@ -54,7 +53,6 @@ backup-maker restore --url $$(cat .build/test/domain.txt) \ -c "cat - > /tmp/test" \ --private-key .build/test/backup.key \ --passphrase riotkit \ - --recipient test@riotkit.org \ --log-level debug ``` @@ -65,25 +63,23 @@ please take a look at `Backup Controller` documentation. **Note: GPG steps are optional** -1. `gpg` keyring is created in a temporary directory, keys are imported +1. `gpg` keys are loaded 2. Command specified in `--cmd` or in `-c` is executed 3. Result of the command, it's stdout is transferred to the `gpg` process 4. From `gpg` process the encoded data is buffered directly to the server 5. Feedback is returned -6. Temporary `gpg` keyring is deleted ## Restore - How it works? It is very similar as in backup operation. -1. `gpg` keyring is created in a temporary directory, keys are imported +1. `gpg` keys are loaded 2. Command specified in `--cmd` or in `-c` is executed 3. `gpg` process is started 4. Backup download is starting 5. Backup is transmitted on the fly from server to `gpg` -> our shell command 6. Our shell `--cmd` / `-c` command is taking stdin and performing a restore action 7. Feedback is returned -8. Temporary `gpg` keyring is deleted ## Automated procedures @@ -96,6 +92,7 @@ together with a tool that generates Backup & Restore procedures. Those procedure - Skip `--private-key` and `--passphrase` to disable GPG - Use `debug` log level to see GPG output and more verbose output at all +- Increase encryption/decryption performance by disabling armoring ## Proposed usage diff --git a/client/download_test.go b/client/download_test.go index 4686163..3859394 100644 --- a/client/download_test.go +++ b/client/download_test.go @@ -90,14 +90,11 @@ func TestDownload_SuccessWithValidGPG(t *testing.T) { ctx := createExampleContext() ctx.ActionType = "download" - ctx.Gpg = context.GPGOperationContext{ + ctx.Crypto = context.EncryptionOperationContext{ PrivateKeyPath: "../resources/test/gpg-key.asc", Passphrase: "riotkit", - Recipient: "test@riotkit.org", + EncType: "gpg-armored", } - initErr := context.InitializeGPGContext(&ctx) - assert.Nil(t, initErr) - defer ctx.Gpg.CleanUp() _ = DownloadBackupIntoProcessStdin(ctx, "cat - > ../.build/TestDownload_SuccessWithValidGPG", client) diff --git a/client/upload.go b/client/upload.go index 2ac9345..1a4699e 100644 --- a/client/upload.go +++ b/client/upload.go @@ -115,7 +115,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error { return pipeErr } - log.Print("Starting cmd.Run()") + log.Print("Starting cmd.Encrypt()") execErr := cmd.Start() if execErr != nil { log.Println("Cannot start backup process ", execErr) @@ -127,6 +127,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error { log.Printf("Starting Upload() for PID=%v", cmd.Process.Pid) status, out, uploadErr := Upload(ctx, client, app.Url, app.CollectionId, app.AuthToken, ReadCloserWithCancellationWhenProcessFails{stdout, cmd, cancel}, app.Timeout) if uploadErr != nil { + cancel() log.Errorf("Status: %v, Out: %v, Err: %v", status, out, uploadErr) return uploadErr } else { diff --git a/client/upload_test.go b/client/upload_test.go index 3f94a45..810eab7 100644 --- a/client/upload_test.go +++ b/client/upload_test.go @@ -25,7 +25,7 @@ func TestUploadFromCommandOutput_PassesOnValidResponse(t *testing.T) { ActionType: "make", VersionToRestore: "", DownloadPath: "", - Gpg: context.GPGOperationContext{}, + Crypto: context.EncryptionOperationContext{}, }, http) assert.Nil(t, err) } @@ -47,7 +47,7 @@ func TestUploadFromCommandOutput_FailOnInvalidResponse(t *testing.T) { ActionType: "make", VersionToRestore: "", DownloadPath: "", - Gpg: context.GPGOperationContext{}, + Crypto: context.EncryptionOperationContext{}, }, http) assert.NotNil(t, err) } diff --git a/cmd/backupmaker/main.go b/cmd/backupmaker/main.go index 89c82b9..fed7680 100644 --- a/cmd/backupmaker/main.go +++ b/cmd/backupmaker/main.go @@ -21,7 +21,7 @@ func addAddGenericFlags(cmd *cobra.Command, ctx *context.Action) { cmd.Flags().StringVarP(&ctx.AuthToken, "auth-token", "t", os.Getenv("BM_AUTH_TOKEN"), "Access token that allows to upload at least one file successfully, [environment variable: BM_AUTH_TOKEN]") cmd.Flags().Int64VarP(&ctx.Timeout, "timeout", "", 60*20, "Connection and read timeout in summary [environment variable: BM_TIMEOUT]") cmd.Flags().StringVarP(&ctx.LogLevelStr, "log-level", "", "info", "Verbosity level: panic|fatal|error|warn|info|debug|trace") - cmd.Flags().BoolVarP(&ctx.Gpg.ShouldShowOutput, "verbose", "", false, "Increase verbosity") + cmd.Flags().StringVarP(&ctx.Crypto.EncType, "encryption-type", "", getEnvOrDefault("BM_ENCRYPTION_TYPE", "gpg-armored"), "Encryption type (options: gpg-armored, gpg-binary) [environment variable: BM_ENCRYPTION_TYPE]") } func createMakeCommand() *cobra.Command { @@ -33,23 +33,19 @@ func createMakeCommand() *cobra.Command { Short: "Upload a backup to remote", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.UploadFromCommandOutput(ctx, client.CreateHttpClient()); err != nil { return err } return nil }, } - makeCmd.Flags().StringVarP(&ctx.Gpg.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]") - makeCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + makeCmd.Flags().StringVarP(&ctx.Crypto.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]") makeCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command to execute, which output will be captured and sent to server [environment variable: BM_CMD]") - makeCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + makeCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + addAddGenericFlags(&makeCmd, &ctx) return &makeCmd @@ -64,13 +60,9 @@ func createRestoreCommand() *cobra.Command { SilenceUsage: true, Short: "Restore a backup from remote to local target", RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.DownloadBackupIntoProcessStdin(ctx, ctx.Command, client.CreateHttpClient()); err != nil { return err } @@ -78,11 +70,11 @@ func createRestoreCommand() *cobra.Command { }, } - restoreCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") + restoreCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") restoreCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command which should take downloaded file as stdin stream e.g. some tar, unzip, psql [environment variable: BM_CMD]") - restoreCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + restoreCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") restoreCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - restoreCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + addAddGenericFlags(&restoreCmd, &ctx) return &restoreCmd } @@ -96,13 +88,9 @@ func createDownloadCommand() *cobra.Command { SilenceUsage: true, Short: "Download a remote backup and print into a local file", RunE: func(cmd *cobra.Command, args []string) error { - defer ctx.Gpg.CleanUp() if err := context.InitializeLogLevel(&ctx); err != nil { return err } - if err := context.InitializeGPGContext(&ctx); err != nil { - return err - } if err := client.DownloadIntoFile(ctx, ctx.DownloadPath, client.CreateHttpClient()); err != nil { return err } @@ -110,11 +98,11 @@ func createDownloadCommand() *cobra.Command { }, } - downloadCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") + downloadCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]") downloadCmd.Flags().StringVarP(&ctx.Command, "save-path", "", os.Getenv("BM_SAVE_PATH"), "Place where to save file instead of executing a restore command [environment variable: BM_SAVE_PATH]") - downloadCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") + downloadCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]") downloadCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]") - downloadCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]") + addAddGenericFlags(&downloadCmd, &ctx) return &downloadCmd } diff --git a/cmd/encryption/main.go b/cmd/encryption/main.go new file mode 100644 index 0000000..06f4432 --- /dev/null +++ b/cmd/encryption/main.go @@ -0,0 +1,90 @@ +package encryption + +import ( + "github.com/pkg/errors" + "github.com/riotkit-org/br-backup-maker/crypto" + "github.com/spf13/cobra" +) + +func NewEncryptionCommand() *cobra.Command { + app := &App{} + + command := &cobra.Command{ + Use: "encrypt", + SilenceUsage: true, + Short: "Encrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return app.Encrypt() + }, + } + + command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file") + command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)") + command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key") + + return command +} + +func NewDecryptionCommand() *cobra.Command { + app := &App{} + + command := &cobra.Command{ + Use: "decrypt", + SilenceUsage: true, + Short: "Decrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return app.Decrypt() + }, + } + + command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file") + command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)") + command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key") + + return command +} + +func NewCryptoCommand() *cobra.Command { + command := &cobra.Command{ + Use: "crypto", + SilenceUsage: true, + Short: "Decrypts a stdin and outputs as stdout", + RunE: func(command *cobra.Command, args []string) error { + return command.Help() + }, + } + command.AddCommand(NewEncryptionCommand()) + command.AddCommand(NewDecryptionCommand()) + return command +} + +type App struct { + keyPath string + encType string + passphrase string +} + +func (encrypt *App) createAlgo() (crypto.Service, error) { + if encrypt.encType == "gpg-armored" { + return crypto.GPGEncryption{Armored: true}, nil + } else if encrypt.encType == "gpg-binary" { + return crypto.GPGEncryption{Armored: false}, nil + } + return nil, errors.New("unsupported encryption type") +} + +func (encrypt *App) Encrypt() error { + algo, err := encrypt.createAlgo() + if err != nil { + return err + } + return algo.Encrypt(encrypt.keyPath, encrypt.passphrase) +} + +func (encrypt *App) Decrypt() error { + algo, err := encrypt.createAlgo() + if err != nil { + return err + } + return algo.Decrypt(encrypt.keyPath, encrypt.passphrase) +} diff --git a/cmd/root.go b/cmd/root.go index fe1a9a5..d8fa19e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,6 +3,7 @@ package cmd import ( "github.com/riotkit-org/br-backup-maker/cmd/backupmaker" "github.com/riotkit-org/br-backup-maker/cmd/bmg" + "github.com/riotkit-org/br-backup-maker/cmd/encryption" "github.com/spf13/cobra" ) @@ -18,5 +19,7 @@ func GetRootCommand() *cobra.Command { cmd.AddCommand(subCmd) } cmd.AddCommand(bmg.CreateCommand()) + cmd.AddCommand(encryption.NewCryptoCommand()) + return cmd } diff --git a/context/action.go b/context/action.go index a92a93d..5b41be5 100644 --- a/context/action.go +++ b/context/action.go @@ -13,7 +13,7 @@ type Action struct { VersionToRestore string DownloadPath string - Gpg GPGOperationContext + Crypto EncryptionOperationContext LogLevel uint32 LogLevelStr string // todo: convert to LogLevel //logLevel, _ := log.ParseLevel(*logLevelStr) @@ -30,21 +30,21 @@ func (that Action) CreateWrappedCommand(custom string) string { cmd = custom } - if !that.Gpg.Enabled(that.ActionType) { + if !that.Crypto.Enabled(that.ActionType) { return cmd } if that.ActionType == "make" { - return cmd + " | " + that.Gpg.GetEncryptionCommand() + return cmd + " | " + that.Crypto.GetEncryptionCommand() } - return that.Gpg.GetDecryptionCommand() + " | " + cmd + return that.Crypto.GetDecryptionCommand() + " | " + cmd } // GetPrintableCommand returns same command as in CreateWrappedCommand(), but with erased credentials // so the command could be logged or printed into the console func (that Action) GetPrintableCommand(custom string) string { - return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Gpg.Passphrase, "***") + return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Crypto.Passphrase, "***") } func (that Action) ShouldShowCommandsOutput() bool { diff --git a/context/action_test.go b/context/action_test.go index 1cec949..e8e4bd8 100644 --- a/context/action_test.go +++ b/context/action_test.go @@ -10,16 +10,15 @@ func TestActionContext_GetCommand_WithGPG(t *testing.T) { ctx := Action{} ctx.Command = "ps aux" ctx.ActionType = "make" - ctx.Gpg = GPGOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + ctx.Crypto = EncryptionOperationContext{ + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "riotkit", + EncType: "gpg-armored", } - assert.Equal(t, "ps aux | gpg --homedir='/path' --encrypt --always-trust --recipient='riotkit@riseup.net' --armor --batch --yes", ctx.CreateWrappedCommand("")) + assert.Contains(t, ctx.CreateWrappedCommand(""), "ps aux | ") + assert.Contains(t, ctx.CreateWrappedCommand(""), "crypto encrypt --key-path=/path/to/key") } // When doing: @@ -29,16 +28,15 @@ func TestActionContext_GetCommand_WithGPG_RestoreActionPlacesPipeAtRightSide(t * ctx := Action{} ctx.Command = "tar xvf -" ctx.ActionType = "restore" - ctx.Gpg = GPGOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + ctx.Crypto = EncryptionOperationContext{ + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "riotkit", + EncType: "gpg-armored", } - assert.Equal(t, "gpg --homedir='/path' --decrypt --recipient='riotkit@riseup.net' --armor --passphrase='riotkit' --batch --yes --pinentry-mode loopback --verbose | tar xvf -", ctx.CreateWrappedCommand("")) + assert.Contains(t, ctx.CreateWrappedCommand(""), "crypto decrypt --key-path=/path/to/key --type=gpg-armored --passphrase='riotkit") + assert.Contains(t, ctx.CreateWrappedCommand(""), "| tar xvf -") } // Check that credentials will be erased @@ -46,13 +44,10 @@ func TestActionContext_GetPrintableCommand_WithGPG(t *testing.T) { ctx := Action{} ctx.Command = "ps aux" ctx.ActionType = "restore" - ctx.Gpg = GPGOperationContext{ - PublicKeyPath: "/path/to/key", - PrivateKeyPath: "/path/to/key", - Passphrase: "my-secret", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + ctx.Crypto = EncryptionOperationContext{ + PublicKeyPath: "/path/to/key", + PrivateKeyPath: "/path/to/key", + Passphrase: "my-secret", } assert.Contains(t, ctx.GetPrintableCommand(""), "--passphrase='***'") @@ -64,13 +59,10 @@ func TestActionContext_GetCommand_PlaintextWithoutEncryption(t *testing.T) { ctx := Action{} ctx.Command = "tar -zcvf - ./" ctx.ActionType = "make" - ctx.Gpg = GPGOperationContext{ - PublicKeyPath: "", - PrivateKeyPath: "", - Passphrase: "riotkit", - Recipient: "riotkit@riseup.net", - Path: "/path", - ShouldShowOutput: false, + ctx.Crypto = EncryptionOperationContext{ + PublicKeyPath: "", + PrivateKeyPath: "", + Passphrase: "riotkit", } assert.Equal(t, "tar -zcvf - ./", ctx.CreateWrappedCommand("")) diff --git a/context/encryption.go b/context/encryption.go new file mode 100644 index 0000000..56e5e2b --- /dev/null +++ b/context/encryption.go @@ -0,0 +1,60 @@ +package context + +import ( + "fmt" + log "github.com/sirupsen/logrus" + "os" + "strings" +) + +type EncryptionOperationContext struct { + PublicKeyPath string + PrivateKeyPath string + Passphrase string + EncType string +} + +func InitializeLogLevel(action *Action) error { + logLevel, parseErr := log.ParseLevel(action.LogLevelStr) + if parseErr != nil { + return parseErr + } + log.SetLevel(logLevel) + action.LogLevel = uint32(logLevel) + return nil +} + +func (that EncryptionOperationContext) GetEncryptionCommand() string { + if that.PublicKeyPath == "" && that.PrivateKeyPath == "" { + log.Debug("No private key, no public key, no encryption then") + return "" + } + return fmt.Sprintf("%s crypto encrypt --key-path=%s --type=%s", that.findBinPath(), that.PublicKeyPath, that.EncType) +} + +func (that EncryptionOperationContext) GetDecryptionCommand() string { + if that.PrivateKeyPath == "" { + log.Debug("No private key, no encryption") + return "" + } + return fmt.Sprintf("%s crypto decrypt --key-path=%s --type=%s --passphrase='%s'", that.findBinPath(), that.PrivateKeyPath, that.EncType, that.Passphrase) +} + +func (that EncryptionOperationContext) findBinPath() string { + executable, _ := os.Executable() + if strings.Contains(executable, ".test") { + return "../.build/backup-maker" + } + if executable == "" { + executable = "backup-maker" + } + return executable +} + +func (that EncryptionOperationContext) Enabled(actionType string) bool { + if actionType == "make" { + return that.PublicKeyPath != "" + } + + return that.PrivateKeyPath != "" || that.PublicKeyPath != "" +} diff --git a/context/encryption_test.go b/context/encryption_test.go new file mode 100644 index 0000000..897b735 --- /dev/null +++ b/context/encryption_test.go @@ -0,0 +1,20 @@ +package context_test + +import ( + "github.com/riotkit-org/br-backup-maker/context" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotReturnCommandWhenGPGDisabled(t *testing.T) { + ctx := context.Action{ + ActionType: "make", + Crypto: context.EncryptionOperationContext{ + PublicKeyPath: "", + PrivateKeyPath: "", + Passphrase: "riotkit", + }, + } + assert.Equal(t, "", ctx.Crypto.GetDecryptionCommand()) + assert.Equal(t, "", ctx.Crypto.GetEncryptionCommand()) +} diff --git a/context/gpg.go b/context/gpg.go deleted file mode 100644 index 7041e5b..0000000 --- a/context/gpg.go +++ /dev/null @@ -1,196 +0,0 @@ -package context - -import ( - "errors" - "fmt" - log "github.com/sirupsen/logrus" - "io" - "os" - "os/exec" -) - -type GPGOperationContext struct { - PublicKeyPath string - PrivateKeyPath string - Passphrase string - Recipient string - - // dynamic - Path string - ShouldShowOutput bool // todo: implement, currently not set, should depend on the verbosity -} - -// InitializeGPGContext is a factory method that creates a GPG directory and imports keys -func InitializeGPGContext(action *Action) error { - ctx := &action.Gpg - path, err := os.MkdirTemp("/tmp", "backup-repository-gpg") - if err != nil { - log.Printf("Cannot create temporary directory for GPG: '%v'", path) - return err - } - if os.Getenv("BR_VERBOSE") == "true" { - ctx.ShouldShowOutput = true - } - ctx.Path = path - if ctx.PublicKeyPath != "" || ctx.PrivateKeyPath != "" { - if initErr := ctx.initializeGPGDirectory(); initErr != nil { - return initErr - } - if importErr := ctx.importKeys(); importErr != nil { - return errors.New(fmt.Sprintf("Cannot import key, error: %v", importErr)) - } - ctx.printImportedKeys() - } else { - log.Warningln("GPG disabled (no keys configured)") - } - return nil -} - -func InitializeLogLevel(action *Action) error { - logLevel, parseErr := log.ParseLevel(action.LogLevelStr) - if parseErr != nil { - return parseErr - } - log.SetLevel(logLevel) - action.LogLevel = uint32(logLevel) - return nil -} - -func (that GPGOperationContext) CleanUp() { - log.Debugf("Cleaning up GPG directory at '%v'", that.Path) - err := os.RemoveAll(that.Path) - - if err != nil { - log.Fatalf("Cannot delete GPG directory at '%v'", that.Path) - } - - _ = exec.Command("/bin/bash", "-c", fmt.Sprintf("ps axu | grep gpg-agent | grep %v | grep -v grep | awk '{print $2}' | xargs kill -9", that.Path)).Run() -} - -// importKeys is importing PUBLIC and PRIVATE keys into local temporary keyring created on-the-fly -func (that GPGOperationContext) importKeys() error { - log.Debugf("Importing GPG keys '%v', '%v'", that.PrivateKeyPath, that.PublicKeyPath) - - for _, keyPath := range []string{that.PrivateKeyPath, that.PublicKeyPath} { - if keyPath == "" { - continue - } - - log.Printf("Importing key %v", keyPath) - cmd := exec.Command( - "gpg", - "--passphrase-fd", "0", - "--pinentry-mode", "loopback", - "--import", keyPath, - ) - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - if that.ShouldShowOutput { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - stdin, _ := cmd.StdinPipe() - _ = cmd.Start() - _, _ = io.WriteString(stdin, that.Passphrase) - _ = stdin.Close() - cmdErr := cmd.Wait() - - if cmdErr != nil { - log.Errorf("Cannot import key '%v'. Check output placed above", keyPath) - return cmdErr - } - } - - return nil -} - -func (that GPGOperationContext) printImportedKeys() { - log.Println("Imported keys:") - cmd := exec.Command("gpg", "--list-secret-keys") - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - _ = cmd.Run() -} - -// initializeGPGDirectory creates a GPG temporary keyring used on-the-fly, then it gets automatically deleted -// by the application -func (that GPGOperationContext) initializeGPGDirectory() error { - log.Println("Initializing GPG directory") - initParamsFilePath := fmt.Sprintf("%v/.init-params", that.Path) - - // create parameters file - _ = os.WriteFile( - initParamsFilePath, - []byte("Key-Type: 1\nKey-Length: 2048\nSubkey-Type: 1\nSubkey-Length: 2048\nName-Real: Backup Maker\nName-Email: riotkit@do-not-use.localhost\nExpire-Date: 0\n"), - 0600, - ) - - cmd := exec.Command( - "gpg", - "--gen-key", - "--passphrase-fd", "0", - "--pinentry-mode", "loopback", - "--batch", initParamsFilePath) - - cmd.Env = []string{fmt.Sprintf("GNUPGHOME=%v", that.Path)} - stdin, err := cmd.StdinPipe() - if that.ShouldShowOutput { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - if err != nil { - _ = cmd.Process.Kill() - return err - } - - runErr := cmd.Start() - if runErr != nil { - return runErr - } - - log.Debugf("Writing Passphrase...") - if _, writeErr := io.WriteString(stdin, that.Passphrase); writeErr != nil { - _ = cmd.Process.Kill() - return writeErr - } - - _ = stdin.Close() - waitErr := cmd.Wait() - if waitErr != nil { - _ = cmd.Process.Kill() - return waitErr - } - - log.Debugf("GPG initialized") - return nil -} - -func (that GPGOperationContext) GetEncryptionCommand() string { - if that.PublicKeyPath == "" && that.PrivateKeyPath == "" { - log.Debug("No private key, no public key, no encryption then") - return "" - } - - return fmt.Sprintf("gpg --homedir='%v' --encrypt --always-trust --recipient='%v' --armor --batch --yes", that.Path, that.Recipient) -} - -func (that GPGOperationContext) GetDecryptionCommand() string { - if that.PrivateKeyPath == "" { - log.Debug("No private key, no encryption") - return "" - } - - return fmt.Sprintf("gpg --homedir='%v' --decrypt --recipient='%v' --armor "+ - "--passphrase='%v' --batch --yes --pinentry-mode loopback --verbose", - that.Path, that.Recipient, that.Passphrase) -} - -func (that GPGOperationContext) Enabled(actionType string) bool { - if actionType == "make" { - return that.PublicKeyPath != "" - } - - return that.PrivateKeyPath != "" || that.PublicKeyPath != "" -} diff --git a/context/gpg_test.go b/context/gpg_test.go deleted file mode 100644 index b7dc023..0000000 --- a/context/gpg_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package context_test - -import ( - "github.com/riotkit-org/br-backup-maker/context" - "github.com/stretchr/testify/assert" - "os/exec" - "testing" -) - -// Success scenario -func TestCreateGPGContext(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "../resources/test/gpg-key.asc", - PrivateKeyPath: "../resources/test/gpg-key.asc", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() - assert.Nil(t, err) -} - -// Import path is not valid. Error message should be in stdout/stderr, in error there could be exit status just -func TestCreateGPGContext_InvalidKeyPath(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "invalid-path", - PrivateKeyPath: "invalid-path", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() - - assert.Equal(t, "Cannot import key, error: exit status 2", err.Error()) -} - -// Encryption is DISABLED as no keys were specified -func TestCreateGPGContext_DisabledEncryption(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "", - PrivateKeyPath: "", - Passphrase: "", - Recipient: "", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() - - assert.Nil(t, err) -} - -// Check that directory and processes are cleaned up (path on disk + gpg-agent process) -func TestGPGOperationContext_CleanUp(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "../resources/test/gpg-key.asc", - PrivateKeyPath: "../resources/test/gpg-key.asc", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() // always clean at the end of the test - assert.Nil(t, err, "Cannot initialize context to verify if its cleaned up later") - - assert.DirExists(t, ctx.Gpg.Path, "Expected the context initialization will create a temporary directory") // GPG temporary directory - ctx.Gpg.CleanUp() - - // GPG temporary directory - assert.NoDirExists(t, ctx.Gpg.Path, "Expected that after calling CleanUp() the temporary directory will no longer exists") - - proc := exec.Command("/bin/bash", "-c", "ps aux |grep %v | grep -v grep") - out, _ := proc.Output() - assert.NotContains(t, string(out), "gpg-agent", "Expected that the gpg-agent will be no longer running as part of the CleanUp() method process") -} - -func TestGPGOperationContext_GetDecryptionCommand_GetEncryptionCommmand_DoesNotReturnCommandWhenGPGDisabled(t *testing.T) { - ctx := context.Action{ - ActionType: "make", - Gpg: context.GPGOperationContext{ - PublicKeyPath: "", - PrivateKeyPath: "", - Passphrase: "riotkit", - Recipient: "example@riotkit.org", - }, - } - err := context.InitializeGPGContext(&ctx) - defer ctx.Gpg.CleanUp() - - assert.Nil(t, err) - assert.Equal(t, "", ctx.Gpg.GetDecryptionCommand()) - assert.Equal(t, "", ctx.Gpg.GetEncryptionCommand()) -} diff --git a/crypto/gpg.go b/crypto/gpg.go new file mode 100644 index 0000000..18187de --- /dev/null +++ b/crypto/gpg.go @@ -0,0 +1,110 @@ +package crypto + +import ( + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "io" + "os" +) + +type GPGEncryption struct { + Armored bool +} + +// readKey is reading and parsing the public/private key +func (GPGEncryption) readKey(keyPath string, passphrase string) (openpgp.EntityList, error) { + f, err := os.Open(keyPath) + if err != nil { + return nil, err + } + defer f.Close() + + keyring, keyringErr := openpgp.ReadArmoredKeyRing(f) + if keyringErr != nil { + return keyring, errors.Wrap(keyringErr, "cannot read keyring (provided key)") + } + + // passphrase is optional + if passphrase != "" { + if err := decryptSecretKey(keyring, passphrase); err != nil { + return keyring, errors.Wrap(err, "cannot decrypt keyring using selected passphrase") + } + } + return keyring, nil +} + +// decryptSecretKey is decrypting all keys in the keyring. Thanks to "wal-g" project maintained by Citus Data Inc. +func decryptSecretKey(entityList openpgp.EntityList, passphrase string) error { + passphraseBytes := []byte(passphrase) + + for _, entity := range entityList { + err := entity.PrivateKey.Decrypt(passphraseBytes) + + if err != nil { + return err + } + + for _, subKey := range entity.Subkeys { + err := subKey.PrivateKey.Decrypt(passphraseBytes) + + if err != nil { + return err + } + } + } + + return nil +} + +// Encrypt is reading from inputStream and writing encrypted stream of bytes to outputStream +func (g GPGEncryption) Encrypt(keyPath string, passphrase string) error { + keyring, keyErr := g.readKey(keyPath, passphrase) + if keyErr != nil { + return errors.Wrap(keyErr, "cannot load public keyring") + } + + // perform encryption + armoredWriter, _ := armor.Encode(os.Stdout, "PGP MESSAGE", nil) + defer armoredWriter.Close() + logrus.Debugln("armor.Encode() setup") + + conversionStream, _ := openpgp.Encrypt(armoredWriter, keyring, nil, &openpgp.FileHints{IsBinary: true}, nil) + defer conversionStream.Close() + logrus.Debugln("openpgp.Encrypt() setup") + + io.Copy(conversionStream, os.Stdin) + logrus.Debugln("io.Copy() finished for GPG encryption") + + return nil +} + +func (g GPGEncryption) Decrypt(keyPath string, passphrase string) error { + // load private key + keyring, keyErr := g.readKey(keyPath, passphrase) + if keyErr != nil { + return errors.Wrap(keyErr, "cannot load public key") + } + + var input io.Reader + input = os.Stdin + + if g.Armored { + armorBlock, armorErr := armor.Decode(os.Stdin) + if armorErr != nil { + return errors.Wrap(armorErr, "cannot setup armor decoder") + } + input = armorBlock.Body + } + + md, readErr := openpgp.ReadMessage(input, keyring, nil, nil) + if readErr != nil { + return errors.Wrap(readErr, "cannot read encrypted contents") + } + if _, err := io.Copy(os.Stdout, md.UnverifiedBody); err != nil { + return errors.Wrap(err, "cannot io.Copy() encrypted stream -> decrypted data") + } + + return nil +} diff --git a/crypto/gpg_test.go b/crypto/gpg_test.go new file mode 100644 index 0000000..d13f361 --- /dev/null +++ b/crypto/gpg_test.go @@ -0,0 +1,20 @@ +package crypto_test + +import ( + "github.com/riotkit-org/br-backup-maker/crypto" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestWithoutValidKey(t *testing.T) { + service := crypto.GPGEncryption{Armored: false} + + assert.NotNil(t, service.Encrypt("gpg_test.go", "test")) + assert.NotNil(t, service.Decrypt("gpg_test.go", "test")) +} + +func TestWithMissingKey(t *testing.T) { + service := crypto.GPGEncryption{Armored: false} + assert.NotNil(t, service.Encrypt("non-existing-path", "test")) + assert.NotNil(t, service.Decrypt("non-existing-path", "test")) +} diff --git a/crypto/interface.go b/crypto/interface.go new file mode 100644 index 0000000..2d51335 --- /dev/null +++ b/crypto/interface.go @@ -0,0 +1,6 @@ +package crypto + +type Service interface { + Encrypt(keyPath string, passphrase string) error + Decrypt(keyPath string, passphrase string) error +} diff --git a/generate/e2e_mysql_test.go b/generate/e2e_mysql_test.go deleted file mode 100644 index 9a7f0da..0000000 --- a/generate/e2e_mysql_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package generate_test - -import ( - "bytes" - "context" - "fmt" - "github.com/pkg/errors" - "github.com/riotkit-org/br-backup-maker/generate" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "os" - "os/exec" - "testing" - "time" -) - -// TestEndToEnd_MariaDBBackupAndRestore an End-To-End testing procedure for MariaDB/MySQL -func TestEndToEnd_MariaDBBackupAndRestoreProcedureBetweenInstances(t *testing.T) { - WithBackupRepositoryDockerStack(func(stack ServiceStack) { - ctx := context.Background() - - // ======================================================== - // Send backup of MariaDB instance #1 to Backup Repository - // ======================================================== - c, dbHostname, dbPort := CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("backup") - subTestMySQLDumpBackup(t, dbHostname, dbPort) - _ = c.Terminate(ctx) - - // ================================================================================= - // Receive Backup from Backup Repository and restore on a new MariaDB instance (#2) - // ================================================================================= - c, dbHostname, dbPort = CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("restore") - subTestMySQLRestoreBackup(t, dbHostname, dbPort) - }) -} - -func TestEndToEnd_MariaDB_AllDatabasesBackup(t *testing.T) { - WithBackupRepositoryDockerStack(func(stack ServiceStack) { - ctx := context.Background() - - // ======================================================== - // Send backup of MariaDB instance #1 to Backup Repository - // ======================================================== - c, dbHostname, dbPort := CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - #db: "" # do not specify the DB parameter, let it be undefined - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("backup") - subTestMySQLDumpBackup(t, dbHostname, dbPort) - _ = c.Terminate(ctx) - - // ================================================================================= - // Receive Backup from Backup Repository and restore on a new MariaDB instance (#2) - // ================================================================================= - c, dbHostname, dbPort = CreateMariaDBContainer(ctx) - time.Sleep(time.Second * 5) - writeDefinitionForLaterSnippetGeneration(` -Params: - hostname: "` + dbHostname + `" - user: "root" - password: "mutual-aid" - port: "` + fmt.Sprintf("%v", dbPort) + `" - db: "" - -Repository: - url: "http://` + stack.ServerHost + `:` + fmt.Sprintf("%v", stack.ServerPort) + `" - token: "` + stack.AdminJwt + `" - encryptionKeyPath: "resources/test/gpg-key.asc" - passphrase: "riotkit" - recipient: "test@riotkit.org" - collectionId: "iwa-ait" - -`) - generateMySQLSnippet("restore") - subTestMySQLRestoreBackup(t, dbHostname, dbPort) - }) -} - -func subTestMySQLDumpBackup(t *testing.T, mysqlHost string, mysqlPort int) { - log.Info("Testing MySQL dump") - - // inject example data - execAndAssert("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "source ../resources/test/mysql-example-structure.sql") - - // verify - sqlCheck := execAndReturn("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "SELECT * FROM Persons;") - assert.Contains(t, sqlCheck, "Bakunin") - - // run backup.sh - cmd := exec.Command("/bin/bash", "-c", "export PATH=$PATH:./; export BR_VERBOSE=true; bash backup.sh 2>&1") - cmd.Dir = "../.build" - cmd.Stderr = os.Stderr - out, err := cmd.Output() - - assert.Nil(t, err, string(out)) - assert.Contains(t, string(out), "Version uploaded") -} - -func subTestMySQLRestoreBackup(t *testing.T, mysqlHost string, mysqlPort int) { - log.Info("Testing MySQL restore") - - // run restore.sh - cmd := exec.Command("/bin/bash", "-c", "export PATH=$PATH:./; bash restore.sh 2>&1") - cmd.Dir = "../.build" - cmd.Stderr = os.Stderr - out, err := cmd.Output() - log.Info("Log from restore: ", string(out)) - - assert.Nil(t, err) - assert.Contains(t, string(out), "Download/restore finished with success") - - // check that data in database exists - `resources/test/mysql-example-structure.sql` inserts a one record with "Mikhail Bakunin" - sqlCheck := execAndReturn("mysql", "-u", "rojava", "-h", mysqlHost, "-projava", "-P", fmt.Sprintf("%v", mysqlPort), "emma_goldman", "-e", "SELECT * FROM Persons;") - assert.Contains(t, sqlCheck, "Bakunin") -} - -func generateMySQLSnippet(operation string) { - bs := generate.SnippetGenerationCommand{ - TemplateName: "mysql-dump-8.0", - DefinitionFile: "../.build/definition.yaml", - IsKubernetes: false, - KeyPath: "../resources/test/gpg-key.asc", - OutputDir: "../.build/", - Schedule: "", - JobName: "", - Image: "", - Operation: operation, - Namespace: "backup-repository", - } - - if err := bs.Run(); err != nil { - log.Fatal(errors.Wrap(err, "Cannot generate backup snippet")) - } -} - -func execAndAssert(command string, args ...string) { - log.Info("execAndAssert", command, args) - cmd := exec.Command(command, args...) - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - if err := cmd.Start(); err != nil { - log.Fatal(errors.Wrap(err, "Failed to start process")) - } - if err := cmd.Wait(); err != nil { - log.Fatal(errors.Wrapf(err, "Process failed: %v. Command: %v %v", buf.String(), command, args)) - } - log.Info(buf.String()) -} - -func execAndReturn(command string, args ...string) string { - log.Info("execAndReturn", command, args) - cmd := exec.Command(command, args...) - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - if err := cmd.Start(); err != nil { - log.Fatal(errors.Wrap(err, "Failed to start process")) - } - if err := cmd.Wait(); err != nil { - log.Fatal(errors.Wrapf(err, "Process failed: %v. Command: %v %v", buf.String(), command, args)) - } - return buf.String() -} diff --git a/generate/templates/backup/mysql-dump.tmpl b/generate/templates/backup/mysql-dump.tmpl index 7b70a9d..c6a7e1c 100644 --- a/generate/templates/backup/mysql-dump.tmpl +++ b/generate/templates/backup/mysql-dump.tmpl @@ -25,5 +25,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/backup/postgres.tmpl b/generate/templates/backup/postgres.tmpl index dafd140..41c2ee1 100644 --- a/generate/templates/backup/postgres.tmpl +++ b/generate/templates/backup/postgres.tmpl @@ -21,5 +21,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/backup/tar.tmpl b/generate/templates/backup/tar.tmpl index 2d73a89..0ab9128 100644 --- a/generate/templates/backup/tar.tmpl +++ b/generate/templates/backup/tar.tmpl @@ -21,5 +21,4 @@ echo " >> Executing backup command and uploading piped result" exec backup-maker make --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info diff --git a/generate/templates/restore/mysql-dump.tmpl b/generate/templates/restore/mysql-dump.tmpl index c5d2b35..ad736a2 100644 --- a/generate/templates/restore/mysql-dump.tmpl +++ b/generate/templates/restore/mysql-dump.tmpl @@ -23,6 +23,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/generate/templates/restore/postgres.tmpl b/generate/templates/restore/postgres.tmpl index c105897..1900aaf 100644 --- a/generate/templates/restore/postgres.tmpl +++ b/generate/templates/restore/postgres.tmpl @@ -17,6 +17,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/generate/templates/restore/tar.tmpl b/generate/templates/restore/tar.tmpl index cad700f..f65b455 100644 --- a/generate/templates/restore/tar.tmpl +++ b/generate/templates/restore/tar.tmpl @@ -17,6 +17,5 @@ echo " >> Restoring version: ${selectedVersion}" exec backup-maker restore --url "{{ .Repository.url }}" \ -c "${COMMAND}" \ --private-key {{ .Repository.encryptionKeyPath }} \ - --recipient {{ .Repository.recipient }}\ --log-level info \ --version=${selectedVersion} diff --git a/go.mod b/go.mod index a69e64c..0dbdd1b 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,11 @@ require ( github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.5 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/containerd v1.6.12 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/go.sum b/go.sum index ccd013b..a713e50 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c h1:3SOlz3Ldp5+/KwuXDbuoj1nWPI6MzqBfCz/KvlPS4ko= +github.com/ProtonMail/go-crypto v0.0.0-20230127150802-22e9f3c8043c/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -111,6 +113,7 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -132,6 +135,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -799,6 +804,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -969,8 +975,10 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml b/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml index c7f0beb..bf56ced 100644 --- a/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml +++ b/resources/filesystem-config/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml @@ -8,8 +8,8 @@ spec: description: IWA-AIT website files filenameTemplate: iwa-ait-${version}.tar.gz maxBackupsCount: 5 - maxOneVersionSize: 1M - maxCollectionSize: 10M + maxOneVersionSize: 1G + maxCollectionSize: 5G # optional windows: diff --git a/resources/test/gpg-key.asc.pub b/resources/test/gpg-key.asc.pub new file mode 100644 index 0000000..a9534c5 --- /dev/null +++ b/resources/test/gpg-key.asc.pub @@ -0,0 +1,51 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGG0nZgBEACyU/y2zIOvBqYTYN/4+QredVIzBCIMoKqPcTRiBncTXgNBCI/7 +1wbVJDyib6Pk1BEaOYnZTG2ewhhO/BxaKs3KK4winmPiHUwFQwKkQaI6McmYjQ0j +JslzUj9poviuVDFf9nctpH3maPxqIwbYmqGwa4aSngEJ0KsDZ8MCZkFeiAOUnbiw +MFoKHxuwJyCQxBtnYTNiO0GgzfJ0gjS0CZlV1fo8FZyQkl1Y4+m695lLPfwHHpiC +ymCK30a2SbQrWiyVOlplnSdJifBEQXv95chNROPoDZ3V9vnujvVydCtV5V1angW5 +mSjBMbWg8uPFuYAi9jj+0ktSs+buqP7MYTEJPleC8A+W7k/YY2JrQRx+yWiMbIYG +6f1LRyljj1vbEfAtDrmBIdIDbVr+F2v7TRAf6PcPPwgv8a9eU8815h5z5Ot9rDIX +P6zuH4ZNJ1L/Tl/h7WST05HaI5OyZ7VNTxbo3q9VBfzxLCyvGE2aLmxwNlf0bKtN +3CkfgBUDbgEiTQsNzSq7BdhJqkMthfMwP/hHIgoQWbtdBfYrjaXyAESw1RYIx1cS +ydacsskCxsg9Q1Z2y4HroaxhvR7kHBEvtdTVtAqJxcBzrtVw9Q/BndbdZeuZla71 +/2elIe+uT/0zMYWHwuP6RBdiV97Ox8hZ3BC5AQNyD0BIWa1cgrODsFe2WQARAQAB +tCN0ZXN0QHJpb3RraXQub3JnIDx0ZXN0QHJpb3RraXQub3JnPokCTgQTAQgAOBYh +BP0I3jy26+HdZrFNraAXQzSce8JyBQJhtJ2YAhsDBQsJCAcCBhUKCQgLAgQWAgMB +Ah4BAheAAAoJEKAXQzSce8Jy2+MP/AwjwDWfQmxt06fq3OZfcF9pNwozerBveMi0 +6neMFdrzgS9nwXbm4xCheXlddfxaPmLejrRv4FlrXwIYvWCXwCXcXopoCJ+9uf/l +RPcw567QqOCJphaifUv155aft3DToGQXsgfF71OLCJpUHR5ETdGQ5H80v5MafOPU +U65w7WEVZQmvD0LY3gFogvcGkB/ft0xOopLQDbeO8qwu9ZFoVTIBaJWFv9P333qo +DQFdzIQAQRiGzsG/bfRRx4jXSGri82ENzFsmCbcNUDxsxu0vL8K60wO/fNMigHzu +7Wbwh1VbkeyEnyvpPA5xwi0lUt1kHYRZwh5eQ5BqLFbJ+NN3hvYyu3JFRbocJJPG +EwqSTzSUV4JtxH3/nH4aQTrhvwmER5vAPKLz0FA8CEeGJ0APMTP0VIMDLyW5IeZa +JjLVuSZ5ugg4UOgGypUYgm6O8kPX9/8Ax0wtkTTA2nXoxj0JMg6UMYyDaAH1Xo+5 +iFDdaEUx2jaJw4uTButheL2MwpQd8LTWJDdk5i15hHXkLz3/jZ5OcCdIrxJagxkY +VlW3S9tR78Br3WUauiFiVBiQohaHXz3OrlTch3Ov+mJzv/UpNIXA+QcjXc6ydSfo +q9Ed1/XAfoITN3i9+Nxy8dqPA6pIGtpAnKphJdFwpWbjL4CIwTU6B/tEf7s3xwCy +3kpWLufJuQINBGG0nZgBEAC+5zkAyFY9TvX+7OLryyg+fDAQC4771iG0ZOsxn92O +Qk9RkgpdjV/hAklxlmHw1Vx+tBwVZgGycZiqr0vkmuJJ73ysjOIjyy0KS+z0jsjg +Kwrhsr4s8Kkd75L5EWKg43SGhHPV3aTw6bPWqN0Ww9FMO0Tmahkt39RFt7Ye79Bv +o3P7WwWOxlB2ABN+DWt4g33wSfaogD4bmsa3Inu7cRUeqB0mLzAi86i6KfQLwl4/ +y/gbeLJTYDJMBznoJQot6Hpu1x5gQYo/2f27tfQSVYOzlZsE2NyCW+LmjUpWnT9+ +kbvy3LweMfGQWPao5p3PEOw0Ee2U4Lz0IaKesaROsB8Hdi+LQiwxNvIXohqEK/tf +6dVvOjNFBQKzituCwjo3JeCc2e/6RsAWq4vImLCAb/uRg3xxQOx/Xw8opB/NkDps +QCVj9RpcrJpnJR/yvGOkb+rkjzTZKdtC4YIncjtcmiQ26sb572YcF+QpaG5yGvim +t16DiS8HOy1dIUXYp/PpdUUgP6sJXWk4YWnAiwMNTb0P68rt/JFeOVFhBQ8YBkZA +/Y88R4TW9hn3L/31ez7ZWhnPu1DWdQx39DXMn3l7npcGn2KU3Qh0UVcF6XEFFeKU +tI/TiksgaI/XkmLmHAW37RNAwxILjEZDNX1NXnPguhWGitkkDlIgHrWxHouYK7Z7 +RwARAQABiQI2BBgBCAAgFiEE/QjePLbr4d1msU2toBdDNJx7wnIFAmG0nZgCGwwA +CgkQoBdDNJx7wnJv5w/+KCdmBZ4EjOhyF3hZ3RozZRepol2oBK0InKx+XkpLNcGb +4FeZPYnX0W6nZPZJFhTN7kTyB4CpKpeVXlnW/FdN0uYB2QkbbNgRInVLXdpBHPXM +2aA9Fy8+ytIx/liNSKRD4hROC+8wBHOmp7m2q6PAoxP4eYWo82eIHc0KWHHsi0Lr +T3Hzk57Cwmcm2rGrmvMnnXoPFl0hMlmFnFXAUxxBoa8yTd9J72S91xGym5+vkRu4 +SvGqCL6tTglq2cp0Lj/JXcvC1tVkkK7H+zK2o8R0qkLVOkufpAx6YMi/9SFIxJGu +SqPoqVMBL9PnK/ml5NCTO7tsjwbPwbDxlXEdVHbhPRk7I6hBxKokkDdKtbVJPSn3 +0yg0tYgjQ1M/DLQvSBYVVZQor3bpSzYCqvSrTFqWWGeLShzSOl75C4hoRKDfZFg3 +r7deE83/zFFZxKPvMysrTrtYwLNGXl1zfkxb15oTYG/fkWH2j7OcES3H9KeWTf1z +/rQ9JY73m3uU1EtMfW6lgrjo1BQ0UzfuePTvjtl+Ma++oqhjDu4oTE3WAAb9bbRE +TjHwq7ejUBWaOH7wfBAUzu7Ee3/o/21HPfBdTxLmGXdQi6uO2e7YMTgGeZVgj2XB +ZfjMSEH23wbGZn8YktFWp2POwa5iAvVFKe1SBmEC8gzg1EwD3DS5lScxDjCISPM= +=U7Ah +-----END PGP PUBLIC KEY BLOCK----- diff --git a/test_backupmaker.mk b/test_backupmaker.mk index ebd4f5b..4a7f896 100644 --- a/test_backupmaker.mk +++ b/test_backupmaker.mk @@ -7,7 +7,6 @@ test_restore: -c "cat - > /tmp/test" \ --private-key .build/test/backup.key \ --passphrase riotkit \ - --recipient test@riotkit.org \ --log-level debug test_create: @@ -17,7 +16,6 @@ test_create: ${BM_BIN_PATH} make --url $$(cat .build/test/domain.txt) \ -c "cat main.go" \ --key .build/test/backup.key \ - --recipient test@riotkit.org \ --log-level debug test_request_cancellation: @@ -27,7 +25,6 @@ test_request_cancellation: ${BM_BIN_PATH} make --url $$(cat .build/test/domain.txt) \ -c "/bin/sh -c 'sleep 1; exit 1'" \ --key gpg-key \ - --recipient test@riotkit.org \ --log-level debug #test_download: diff --git a/versions.mk b/versions.mk index 9abbe22..e3f625d 100644 --- a/versions.mk +++ b/versions.mk @@ -3,7 +3,7 @@ # # Used in E2E testing -TEST_BACKUP_REPOSITORY_VERSION="v4.0.0-rc13" +TEST_BACKUP_REPOSITORY_VERSION="v4.0.0" TEST_POSTGRES_VERSION="14.2-alpine" TEST_MINIO_VERSION="2022.4.8-debian-10-r0" TEST_MARIADB_VERSION="10.7.3-focal"