Initial encryption support

Makefile

@@ -5,11 +5,15 @@ DOCKER_TAG := $(or ${GITHUB_TAG_NAME}, latest)
 BACKUP_PROVIDER := $(or ${BACKUP_PROVIDER},local)
 
 .PHONY: all
-all:
+all: test
 	go mod tidy
 	go build -ldflags "$(LINKMODE)" -tags 'osusergo netgo static_build' -o bin/backup-restore-sidecar github.com/metal-stack/backup-restore-sidecar/cmd
 	strip bin/backup-restore-sidecar
 
+.PHONY: test
+test:
+	go test -race ./...
+
 .PHONY: proto
 proto:
 	docker run -it --rm -v ${PWD}/api:/work/api metalstack/builder protoc -I api/ api/v1/*.proto --go_out=plugins=grpc:api

README.md

@@ -29,6 +29,12 @@ With `--compression-method` you can define how generated backups are compressed
 - S3 Buckets (tested against Ceph RADOS gateway)
 - Local
 
+## Encryption
+
+For all three storage providers are AES encryption is supported, can be enabled with `--encryption-key=<YOUR_KEY>`.
+The key must be either 16 bytes (AES-128), 24 bytes (AES-192) or 32 bytes (AES-256) long.
+The backups are stored at the storage provider with the `.aes` suffix. If the file does not have this suffix, decryption is skipped.
+
 ## How it works
 
 ![Sequence Diagram](docs/sequence.png)

cmd/internal/backup/backup.go

@@ -8,13 +8,14 @@ import (
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/constants"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database"
+	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics"
 	cron "github.com/robfig/cron/v3"
 	"go.uber.org/zap"
 )
 
 // Start starts the backup component, which is periodically taking backups of the database
-func Start(log *zap.SugaredLogger, backupSchedule string, db database.DatabaseProber, bp backuproviders.BackupProvider, metrics *metrics.Metrics, comp *compress.Compressor, stop <-chan struct{}) error {
+func Start(log *zap.SugaredLogger, backupSchedule string, db database.DatabaseProber, bp backuproviders.BackupProvider, metrics *metrics.Metrics, comp *compress.Compressor, encrypter *encryption.Encrypter, stop <-chan struct{}) error {
 	log.Info("database is now available, starting periodic backups")
 
 	c := cron.New()
@@ -45,6 +46,15 @@ func Start(log *zap.SugaredLogger, backupSchedule string, db database.DatabasePr
 		}
 		log.Info("compressed backup")
 
+		if encrypter != nil {
+			filename, err = encrypter.Encrypt(filename)
+			if err != nil {
+				metrics.CountError("encrypt")
+				log.Errorw("unable to encrypt backup", "error", err)
+				return
+			}
+		}
+
 		err = bp.UploadBackup(filename)
 		if err != nil {
 			metrics.CountError("upload")

cmd/internal/backup/providers/gcp/gcp.go

@@ -162,7 +162,6 @@ func (b *BackupProviderGCP) DownloadBackup(version *providers.BackupVersion) err
 	if err != nil {
 		return errors.Wrap(err, "error writing file from gcp to filesystem")
 	}
-
 	return nil
 }
 

cmd/internal/backup/providers/local/local.go

@@ -82,6 +82,7 @@ func (b *BackupProviderLocal) DownloadBackup(version *providers.BackupVersion) e
 	b.log.Infow("download backup called for provider local")
 	source := filepath.Join(b.config.LocalBackupPath, version.Name)
 	destination := filepath.Join(constants.DownloadDir, version.Name)
+
 	err := utils.Copy(source, destination)
 	if err != nil {
 		return err

cmd/internal/backup/providers/s3/s3.go

@@ -207,7 +207,6 @@ func (b *BackupProviderS3) DownloadBackup(version *providers.BackupVersion) erro
 	if err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -237,7 +236,6 @@ func (b *BackupProviderS3) UploadBackup(sourcePath string) error {
 	if err != nil {
 		return err
 	}
-
 	return nil
 }
 

cmd/internal/encryption/encryption.go

@@ -0,0 +1,191 @@
+package encryption
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode"
+
+	"go.uber.org/zap"
+)
+
+// Suffix is appended on encryption and removed on decryption from given input
+const Suffix = ".aes"
+
+// Encrypter is used to encrypt/decrypt backups
+type Encrypter struct {
+	key string
+	log *zap.SugaredLogger
+}
+
+// New creates a new Encrypter with the given key.
+// The key should be 16 bytes (AES-128), 24 bytes (AES-192) or
+// 32 bytes (AES-256)
+func New(log *zap.SugaredLogger, key string) (*Encrypter, error) {
+	switch len(key) {
+	case 16, 24, 32:
+	default:
+		return nil, fmt.Errorf("key length:%d invalid, must be 16,24 or 32 bytes", len(key))
+	}
+	if !isASCII(key) {
+		return nil, fmt.Errorf("key must only contain ascii characters")
+	}
+
+	return &Encrypter{
+		key: key,
+		log: log,
+	}, nil
+}
+
+// Encrypt input file with key and store the encrypted files with suffix appended
+func (e *Encrypter) Encrypt(input string) (string, error) {
+	output := input + Suffix
+	e.log.Infow("encrypt", "input", input, "output", output)
+	infile, err := os.Open(input)
+	if err != nil {
+		return "", err
+	}
+	defer infile.Close()
+
+	key := []byte(e.key)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	// Never use more than 2^32 random nonces with a given key
+	// because of the risk of repeat.
+	iv := make([]byte, block.BlockSize())
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return "", err
+	}
+
+	outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777)
+	if err != nil {
+		return "", err
+	}
+	defer outfile.Close()
+
+	// The buffer size must be multiple of 16 bytes
+	buf := make([]byte, 1024)
+	stream := cipher.NewCTR(block, iv)
+	for {
+		n, err := infile.Read(buf)
+		if n > 0 {
+			stream.XORKeyStream(buf, buf[:n])
+			// Write into file
+			_, err = outfile.Write(buf[:n])
+			if err != nil {
+				return "", err
+			}
+		}
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			e.log.Infof("Read %d bytes: %v", n, err)
+			break
+		}
+	}
+	// Append the IV
+	_, err = outfile.Write(iv)
+	if err == nil {
+		err := os.Remove(input)
+		if err != nil {
+			e.log.Warnw("unable to remove input", "error", err)
+		}
+	}
+	return output, err
+}
+
+// Decrypt input file with key and store decrypted result with suffix removed
+// if input does not end with suffix, it is assumed that the file was not encrypted.
+func (e *Encrypter) Decrypt(input string) (string, error) {
+	output := strings.TrimSuffix(input, Suffix)
+	e.log.Infow("decrypt", "input", input, "output", output)
+	extension := filepath.Ext(input)
+	if extension != Suffix {
+		return input, fmt.Errorf("input is not encrypted")
+	}
+	infile, err := os.Open(input)
+	if err != nil {
+		return "", err
+	}
+	defer infile.Close()
+
+	key := []byte(e.key)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	// Never use more than 2^32 random nonces with a given key
+	// because of the risk of repeat.
+	fi, err := infile.Stat()
+	if err != nil {
+		return "", err
+	}
+
+	iv := make([]byte, block.BlockSize())
+	msgLen := fi.Size() - int64(len(iv))
+	_, err = infile.ReadAt(iv, msgLen)
+	if err != nil {
+		return "", err
+	}
+
+	outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777)
+	if err != nil {
+		return "", err
+	}
+	defer outfile.Close()
+
+	// The buffer size must be multiple of 16 bytes
+	buf := make([]byte, 1024)
+	stream := cipher.NewCTR(block, iv)
+	for {
+		n, err := infile.Read(buf)
+		if n > 0 {
+			// The last bytes are the IV, don't belong the original message
+			if n > int(msgLen) {
+				n = int(msgLen)
+			}
+			msgLen -= int64(n)
+			stream.XORKeyStream(buf, buf[:n])
+			// Write into file
+			_, err = outfile.Write(buf[:n])
+			if err != nil {
+				return "", err
+			}
+		}
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			e.log.Infof("Read %d bytes: %v", n, err)
+			break
+		}
+	}
+	err = os.Remove(input)
+	if err != nil {
+		e.log.Warnw("unable to remove input", "error", err)
+	}
+	return output, nil
+}
+
+func isASCII(s string) bool {
+	for _, c := range s {
+		if c > unicode.MaxASCII {
+			return false
+		}
+	}
+	return true
+}

cmd/internal/encryption/encryption_test.go

@@ -0,0 +1,58 @@
+package encryption
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+)
+
+func TestEncrypter(t *testing.T) {
+	// Key too short
+	_, err := New(zap.L().Sugar(), "tooshortkey")
+	assert.EqualError(t, err, "key length:11 invalid, must be 16,24 or 32 bytes")
+
+	// Key too short
+	_, err = New(zap.L().Sugar(), "19bytesofencryption")
+	assert.EqualError(t, err, "key length:19 invalid, must be 16,24 or 32 bytes")
+
+	// Key too long
+	_, err = New(zap.L().Sugar(), "tooloooonoooooooooooooooooooooooooooongkey")
+	assert.EqualError(t, err, "key length:42 invalid, must be 16,24 or 32 bytes")
+
+	e, err := New(zap.L().Sugar(), "01234567891234560123456789123456")
+	assert.NoError(t, err, "")
+
+	input, err := ioutil.TempFile("", "encrypt")
+	assert.NoError(t, err)
+	defer os.Remove(input.Name())
+
+	cleartextInput := []byte("This is the content of the file")
+	err = ioutil.WriteFile(input.Name(), cleartextInput, 0644)
+	assert.NoError(t, err)
+	output, err := e.Encrypt(input.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, input.Name()+Suffix, output)
+
+	cleartextFile, err := e.Decrypt(output)
+	assert.NoError(t, err)
+	cleartext, err := ioutil.ReadFile(cleartextFile)
+	assert.NoError(t, err)
+	assert.Equal(t, cleartextInput, cleartext)
+
+	// Test with 100MB file
+	bigBuff := make([]byte, 100000000)
+	err = ioutil.WriteFile("bigfile.test", bigBuff, 0666)
+	assert.NoError(t, err)
+
+	bigEncFile, err := e.Encrypt("bigfile.test")
+	assert.NoError(t, err)
+	_, err = e.Decrypt(bigEncFile)
+	assert.NoError(t, err)
+	os.Remove("bigfile.test")
+	os.Remove("bigfile.test.aes")
+
+}