Merge branch 'master' of github.com:mozilla/sops

CHANGELOG.rst

@@ -1,6 +1,35 @@
 Changelog
 =========
 
+3.0.0
+-----
+
+* Shamir secret sharing scheme support allows SOPS to require multiple master
+  keys to access a data key and decrypt a file. See `sops groups -help` and the
+  documentation in README.
+
+* Keyservice to forward access to a local master key on a socket, similar to
+  gpg-agent. See `sops keyservice --help` and the documentation in README.
+
+* Encrypt comments by default
+
+* Support for Google Compute Platform KMS
+
+* Refactor of the store logic to separate the internal representation SOPS
+  has of files from the external representation used in JSON and YAML files
+
+* Reencoding of versions as string on sops 1.X files.
+  **WARNING** this change breaks backward compatibility.
+  SOPS shows an error message with instructions on how to solve
+  this if it happens.
+
+* Added command to reconfigure the keys used to encrypt/decrypt a file based on the .sops.yaml config file
+
+* Retrieve missing PGP keys from gpg.mozilla.org
+
+* Improved error messages for errors when decrypting files
+
+
 2.0.0
 -----
 

cmd/sops/common/common.go

@@ -32,11 +32,11 @@ type DecryptTreeOpts struct {
 func DecryptTree(opts DecryptTreeOpts) (dataKey []byte, err error) {
 	dataKey, err = opts.Tree.Metadata.GetDataKeyWithKeyServices(opts.KeyServices)
 	if err != nil {
-		return nil, cli.NewExitError(err.Error(), codes.CouldNotRetrieveKey)
+		return nil, NewExitError(err, codes.CouldNotRetrieveKey)
 	}
 	computedMac, err := opts.Tree.Decrypt(dataKey, opts.Cipher)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error decrypting tree: %s", err), codes.ErrorDecryptingTree)
+		return nil, NewExitError(fmt.Sprintf("Error decrypting tree: %s", err), codes.ErrorDecryptingTree)
 	}
 	fileMac, err := opts.Cipher.Decrypt(opts.Tree.Metadata.MessageAuthenticationCode, dataKey, opts.Tree.Metadata.LastModified.Format(time.RFC3339))
 	if !opts.IgnoreMac {
@@ -45,7 +45,7 @@ func DecryptTree(opts DecryptTreeOpts) (dataKey []byte, err error) {
 			if fileMac == "" {
 				fileMac = "no MAC"
 			}
-			return nil, cli.NewExitError(fmt.Sprintf("MAC mismatch. File has %s, computed %s", fileMac, computedMac), codes.MacMismatch)
+			return nil, NewExitError(fmt.Sprintf("MAC mismatch. File has %s, computed %s", fileMac, computedMac), codes.MacMismatch)
 		}
 	}
 	return dataKey, nil
@@ -65,12 +65,12 @@ type EncryptTreeOpts struct {
 func EncryptTree(opts EncryptTreeOpts) error {
 	unencryptedMac, err := opts.Tree.Encrypt(opts.DataKey, opts.Cipher)
 	if err != nil {
-		return cli.NewExitError(fmt.Sprintf("Error encrypting tree: %s", err), codes.ErrorEncryptingTree)
+		return NewExitError(fmt.Sprintf("Error encrypting tree: %s", err), codes.ErrorEncryptingTree)
 	}
 	opts.Tree.Metadata.LastModified = time.Now().UTC()
 	opts.Tree.Metadata.MessageAuthenticationCode, err = opts.Cipher.Encrypt(unencryptedMac, opts.DataKey, opts.Tree.Metadata.LastModified.Format(time.RFC3339))
 	if err != nil {
-		return cli.NewExitError(fmt.Sprintf("Could not encrypt MAC: %s", err), codes.ErrorEncryptingMac)
+		return NewExitError(fmt.Sprintf("Could not encrypt MAC: %s", err), codes.ErrorEncryptingMac)
 	}
 	return nil
 }
@@ -79,15 +79,15 @@ func EncryptTree(opts EncryptTreeOpts) error {
 func LoadEncryptedFile(inputStore sops.Store, inputPath string) (*sops.Tree, error) {
 	fileBytes, err := ioutil.ReadFile(inputPath)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error reading file: %s", err), codes.CouldNotReadInputFile)
+		return nil, NewExitError(fmt.Sprintf("Error reading file: %s", err), codes.CouldNotReadInputFile)
 	}
 	metadata, err := inputStore.UnmarshalMetadata(fileBytes)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error loading file metadata: %s", err), codes.CouldNotReadInputFile)
+		return nil, NewExitError(fmt.Sprintf("Error loading file metadata: %s", err), codes.CouldNotReadInputFile)
 	}
 	branch, err := inputStore.Unmarshal(fileBytes)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error loading file: %s", err), codes.CouldNotReadInputFile)
+		return nil, NewExitError(fmt.Sprintf("Error loading file: %s", err), codes.CouldNotReadInputFile)
 	}
 	tree := sops.Tree{
 		Branch:   branch,
@@ -96,6 +96,13 @@ func LoadEncryptedFile(inputStore sops.Store, inputPath string) (*sops.Tree, err
 	return &tree, nil
 }
 
+func NewExitError(i interface{}, exitCode int) *cli.ExitError {
+	if userErr, ok := i.(sops.UserError); ok {
+		return NewExitError(userErr.UserError(), exitCode)
+	}
+	return cli.NewExitError(i, exitCode)
+}
+
 func IsYAMLFile(path string) bool {
 	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
 }

cmd/sops/decrypt.go

@@ -7,7 +7,6 @@ import (
 	"go.mozilla.org/sops/cmd/sops/codes"
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keyservice"
-	"gopkg.in/urfave/cli.v1"
 )
 
 type decryptOpts struct {
@@ -41,7 +40,7 @@ func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
 	}
 	decryptedFile, err = opts.OutputStore.Marshal(tree.Branch)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
 	}
 	return decryptedFile, err
 }
@@ -55,13 +54,13 @@ func extract(tree *sops.Tree, path []interface{}, outputStore sops.Store) (outpu
 		tree.Branch = newBranch
 		decrypted, err := outputStore.Marshal(tree.Branch)
 		if err != nil {
-			return nil, cli.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
+			return nil, common.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
 		}
 		return decrypted, err
 	}
 	bytes, err := outputStore.MarshalValue(v)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error dumping tree: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Error dumping tree: %s", err), codes.ErrorDumpingTree)
 	}
 	return bytes, nil
 }

cmd/sops/edit.go

@@ -21,7 +21,6 @@ import (
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keyservice"
 	"go.mozilla.org/sops/stores/json"
-	"gopkg.in/urfave/cli.v1"
 )
 
 type editOpts struct {
@@ -91,7 +90,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 	var tree sops.Tree
 	branch, err := opts.InputStore.Unmarshal(fileBytes)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
 	}
 	tree.Branch = branch
 	tree.Metadata = sops.Metadata{
@@ -104,7 +103,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 	// Generate a data key
 	dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices)
 	if len(errs) > 0 {
-		return nil, cli.NewExitError(fmt.Sprintf("Error encrypting the data key with one or more master keys: %s", errs), codes.CouldNotRetrieveKey)
+		return nil, common.NewExitError(fmt.Sprintf("Error encrypting the data key with one or more master keys: %s", errs), codes.CouldNotRetrieveKey)
 	}
 
 	return editTree(opts.editOpts, &tree, dataKey)
@@ -131,12 +130,12 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 	// Create temporary file for editing
 	tmpdir, err := ioutil.TempDir("", "")
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not create temporary directory: %s", err), codes.CouldNotWriteOutputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Could not create temporary directory: %s", err), codes.CouldNotWriteOutputFile)
 	}
 	defer os.RemoveAll(tmpdir)
 	tmpfile, err := os.Create(path.Join(tmpdir, path.Base(opts.InputPath)))
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not create temporary file: %s", err), codes.CouldNotWriteOutputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Could not create temporary file: %s", err), codes.CouldNotWriteOutputFile)
 	}
 
 	// Write to temporary file
@@ -147,17 +146,17 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 		out, err = opts.OutputStore.Marshal(tree.Branch)
 	}
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
 	}
 	_, err = tmpfile.Write(out)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
 	}
 
 	// Compute file hash to detect if the file has been edited
 	origHash, err := hashFile(tmpfile.Name())
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not hash file: %s", err), codes.CouldNotReadInputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Could not hash file: %s", err), codes.CouldNotReadInputFile)
 	}
 
 	// Let the user edit the file
@@ -179,7 +178,7 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 	// Output the file
 	encryptedFile, err := opts.OutputStore.MarshalWithMetadata(tree.Branch, tree.Metadata)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
 	}
 	return encryptedFile, nil
 }
@@ -188,18 +187,18 @@ func runEditorUntilOk(opts runEditorUntilOkOpts) error {
 	for {
 		err := runEditor(opts.TmpFile.Name())
 		if err != nil {
-			return cli.NewExitError(fmt.Sprintf("Could not run editor: %s", err), codes.NoEditorFound)
+			return common.NewExitError(fmt.Sprintf("Could not run editor: %s", err), codes.NoEditorFound)
 		}
 		newHash, err := hashFile(opts.TmpFile.Name())
 		if err != nil {
-			return cli.NewExitError(fmt.Sprintf("Could not hash file: %s", err), codes.CouldNotReadInputFile)
+			return common.NewExitError(fmt.Sprintf("Could not hash file: %s", err), codes.CouldNotReadInputFile)
 		}
 		if bytes.Equal(newHash, opts.OriginalHash) {
-			return cli.NewExitError("File has not changed, exiting.", codes.FileHasNotBeenModified)
+			return common.NewExitError("File has not changed, exiting.", codes.FileHasNotBeenModified)
 		}
 		edited, err := ioutil.ReadFile(opts.TmpFile.Name())
 		if err != nil {
-			return cli.NewExitError(fmt.Sprintf("Could not read edited file: %s", err), codes.CouldNotReadInputFile)
+			return common.NewExitError(fmt.Sprintf("Could not read edited file: %s", err), codes.CouldNotReadInputFile)
 		}
 		newBranch, err := opts.InputStore.Unmarshal(edited)
 		if err != nil {
@@ -228,7 +227,7 @@ func runEditorUntilOk(opts runEditorUntilOkOpts) error {
 		opts.Tree.Branch = newBranch
 		needVersionUpdated, err := AIsNewerThanB(version, opts.Tree.Metadata.Version)
 		if err != nil {
-			return cli.NewExitError(fmt.Sprintf("Failed to compare document version %q with program version %q: %v", opts.Tree.Metadata.Version, version, err), codes.FailedToCompareVersions)
+			return common.NewExitError(fmt.Sprintf("Failed to compare document version %q with program version %q: %v", opts.Tree.Metadata.Version, version, err), codes.FailedToCompareVersions)
 		}
 		if needVersionUpdated {
 			opts.Tree.Metadata.Version = version

cmd/sops/encrypt.go

@@ -9,7 +9,6 @@ import (
 	"go.mozilla.org/sops/cmd/sops/codes"
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keyservice"
-	"gopkg.in/urfave/cli.v1"
 )
 
 type encryptOpts struct {
@@ -27,12 +26,12 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
 	// Load the file
 	fileBytes, err := ioutil.ReadFile(opts.InputPath)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error reading file: %s", err), codes.CouldNotReadInputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Error reading file: %s", err), codes.CouldNotReadInputFile)
 	}
 	var tree sops.Tree
 	branch, err := opts.InputStore.Unmarshal(fileBytes)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
+		return nil, common.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
 	}
 	tree.Branch = branch
 	tree.Metadata = sops.Metadata{
@@ -58,7 +57,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
 
 	encryptedFile, err = opts.OutputStore.MarshalWithMetadata(tree.Branch, tree.Metadata)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
 	}
 	return
 }

cmd/sops/main.go

@@ -226,10 +226,10 @@ func main() {
 			Action: func(c *cli.Context) error {
 				configPath, err := config.FindConfigFile(".")
 				if err != nil {
-					return cli.NewExitError(err, codes.ErrorGeneric)
+					return common.NewExitError(err, codes.ErrorGeneric)
 				}
 				if c.NArg() < 1 {
-					return cli.NewExitError("Error: no file specified", codes.NoFileSpecified)
+					return common.NewExitError("Error: no file specified", codes.NoFileSpecified)
 				}
 				err = updatekeys.UpdateKeys(updatekeys.Opts{
 					InputPath:   c.Args()[0],
@@ -241,7 +241,7 @@ func main() {
 				if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil {
 					return cliErr
 				} else if err != nil {
-					return cli.NewExitError(err, codes.ErrorGeneric)
+					return common.NewExitError(err, codes.ErrorGeneric)
 				}
 				return nil
 			},
@@ -348,15 +348,15 @@ func main() {
 
 	app.Action = func(c *cli.Context) error {
 		if c.NArg() < 1 {
-			return cli.NewExitError("Error: no file specified", codes.NoFileSpecified)
+			return common.NewExitError("Error: no file specified", codes.NoFileSpecified)
 		}
 		fileName := c.Args()[0]
 		if _, err := os.Stat(fileName); os.IsNotExist(err) {
 			if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" {
-				return cli.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", 49)
+				return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", 49)
 			}
 			if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") {
-				return cli.NewExitError("Error: cannot operate on non-existent file", codes.NoFileSpecified)
+				return common.NewExitError("Error: cannot operate on non-existent file", codes.NoFileSpecified)
 			}
 		}
 
@@ -393,7 +393,7 @@ func main() {
 			var extract []interface{}
 			extract, err = parseTreePath(c.String("extract"))
 			if err != nil {
-				return cli.NewExitError(fmt.Errorf("error parsing --extract path: %s", err), codes.InvalidTreePathFormat)
+				return common.NewExitError(fmt.Errorf("error parsing --extract path: %s", err), codes.InvalidTreePathFormat)
 			}
 			output, err = decrypt(decryptOpts{
 				OutputStore: outputStore,
@@ -503,7 +503,7 @@ func main() {
 		if c.Bool("in-place") || isEditMode || c.String("set") != "" {
 			file, err := os.Create(fileName)
 			if err != nil {
-				return cli.NewExitError(fmt.Sprintf("Could not open in-place file for writing: %s", err), codes.CouldNotWriteOutputFile)
+				return common.NewExitError(fmt.Sprintf("Could not open in-place file for writing: %s", err), codes.CouldNotWriteOutputFile)
 			}
 			defer file.Close()
 			_, err = file.Write(output)
@@ -617,7 +617,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) {
 	var cloudKmsKeys []keys.MasterKey
 	kmsEncryptionContext := kms.ParseKMSContext(c.String("encryption-context"))
 	if c.String("encryption-context") != "" && kmsEncryptionContext == nil {
-		return nil, cli.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat)
+		return nil, common.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat)
 	}
 	if c.String("kms") != "" {
 		for _, k := range kms.MasterKeysFromArnString(c.String("kms"), kmsEncryptionContext) {
@@ -685,7 +685,7 @@ func jsonValueToTreeInsertableValue(jsonValue string) (interface{}, error) {
 	var valueToInsert interface{}
 	err := encodingjson.Unmarshal([]byte(jsonValue), &valueToInsert)
 	if err != nil {
-		return nil, cli.NewExitError("Value for --set is not valid JSON", codes.ErrorInvalidSetFormat)
+		return nil, common.NewExitError("Value for --set is not valid JSON", codes.ErrorInvalidSetFormat)
 	}
 	// Check if decoding it as json we find a single value
 	// and not a map or slice, in which case we can't marshal
@@ -695,7 +695,7 @@ func jsonValueToTreeInsertableValue(jsonValue string) (interface{}, error) {
 		var err error
 		valueToInsert, err = (&json.Store{}).Unmarshal([]byte(jsonValue))
 		if err != nil {
-			return nil, cli.NewExitError("Invalid --set value format", codes.ErrorInvalidSetFormat)
+			return nil, common.NewExitError("Invalid --set value format", codes.ErrorInvalidSetFormat)
 		}
 	}
 	return valueToInsert, nil
@@ -706,15 +706,15 @@ func extractSetArguments(set string) (path []interface{}, valueToInsert interfac
 	// Since python-dict-index has to end with ], we split at "] " to get the two parts
 	pathValuePair := strings.SplitAfterN(set, "] ", 2)
 	if len(pathValuePair) < 2 {
-		return nil, nil, cli.NewExitError("Invalid --set format", codes.ErrorInvalidSetFormat)
+		return nil, nil, common.NewExitError("Invalid --set format", codes.ErrorInvalidSetFormat)
 	}
 	fullPath := strings.TrimRight(pathValuePair[0], " ")
 	jsonValue := pathValuePair[1]
 	valueToInsert, err = jsonValueToTreeInsertableValue(jsonValue)
 
 	path, err = parseTreePath(fullPath)
 	if err != nil {
-		return nil, nil, cli.NewExitError("Invalid --set format", codes.ErrorInvalidSetFormat)
+		return nil, nil, common.NewExitError("Invalid --set format", codes.ErrorInvalidSetFormat)
 	}
 	return path, valueToInsert, nil
 }

cmd/sops/rotate.go

@@ -8,7 +8,6 @@ import (
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keys"
 	"go.mozilla.org/sops/keyservice"
-	cli "gopkg.in/urfave/cli.v1"
 )
 
 type rotateOpts struct {
@@ -68,7 +67,7 @@ func rotate(opts rotateOpts) ([]byte, error) {
 
 	encryptedFile, err := opts.OutputStore.MarshalWithMetadata(tree.Branch, tree.Metadata)
 	if err != nil {
-		return nil, cli.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+		return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
 	}
 	return encryptedFile, nil
 }