Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow multiple sopsFiles for easier common secrets sharing across multiple configuration #417

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Allow multiple sopsFiles for easier common secrets sharing across multiple configuration #417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
262a222
feat(shadowing): golang
vdbe Feb 2, 2024
8ee9a71
feat(shadowing): nixos module
vdbe Feb 2, 2024
0068d61
feat(shadowing): home manager module
vdbe Feb 2, 2024
c623385
test(shadowing): sops-install-secrets
vdbe Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 40 additions & 12 deletions modules/home-manager/sops.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@
let
cfg = config.sops;
sops-install-secrets = (pkgs.callPackage ../.. {}).sops-install-secrets;
secretType = lib.types.submodule ({ config, name, ... }: {
secretType = lib.types.submodule ({ config, options, name, ... }: {
config = lib.mkMerge[{
sopsFile = lib.mkOptionDefault cfg.defaultSopsFile;
sopsFiles = lib.mkIf (lib.length cfg.defaultSopsFiles > 0) (lib.mkOptionDefault cfg.defaultSopsFiles);
}
{
sopsFiles = lib.mkIf (config.sopsFile != null) ( lib.mkOverride options.sopsFile.highestPrio (lib.mkBefore [config.sopsFile]));
}];
options = {
name = lib.mkOption {
type = lib.types.str;
Expand Down Expand Up @@ -53,12 +60,19 @@ let

sopsFile = lib.mkOption {
type = lib.types.path;
default = cfg.defaultSopsFile;
defaultText = "\${config.sops.defaultSopsFile}";
description = ''
Sops file the secret is loaded from.
'';
};

sopsFiles = lib.mkOption {
type = lib.types.nonEmptyListOf lib.types.path;
defaultText = "\${config.sops.defaultSopsFiles}";
description = ''
Sops files the secret is loaded from.
'';
};
};
});

Expand Down Expand Up @@ -110,12 +124,21 @@ in {
};

defaultSopsFile = lib.mkOption {
type = lib.types.path;
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
Default sops file used for all secrets.
'';
};

defaultSopsFiles = lib.mkOption {
type = lib.types.listOf lib.types.path;
default = [];
description = ''
Default sops files used for all secrets.
'';
};

defaultSopsFormat = lib.mkOption {
type = lib.types.str;
default = "yaml";
Expand Down Expand Up @@ -222,15 +245,20 @@ in {
assertion = !(cfg.gnupg.home != null && cfg.gnupg.sshKeyPaths != []);
message = "Exactly one of sops.gnupg.home and sops.gnupg.sshKeyPaths must be set";
}] ++ lib.optionals cfg.validateSopsFiles (
lib.concatLists (lib.mapAttrsToList (name: secret: [{
assertion = builtins.pathExists secret.sopsFile;
message = "Cannot find path '${secret.sopsFile}' set in sops.secrets.${lib.strings.escapeNixIdentifier name}.sopsFile";
} {
assertion =
builtins.isPath secret.sopsFile ||
(builtins.isString secret.sopsFile && lib.hasPrefix builtins.storeDir secret.sopsFile);
message = "'${secret.sopsFile}' is not in the Nix store. Either add it to the Nix store or set sops.validateSopsFiles to false";
}]) cfg.secrets)
lib.concatLists (lib.mapAttrsToList
(name: secret:
lib.concatMap
(sopsFile: [{
assertion = builtins.pathExists sopsFile;
message = "Cannot find path '${sopsFile}' set in sops.secrets.${lib.strings.escapeNixIdentifier name}.sopsFiles";
} {
assertion =
builtins.isPath sopsFile ||
(builtins.isString sopsFile && lib.hasPrefix builtins.storeDir sopsFile);
message = "'${sopsFile}' is not in the Nix store. Either add it to the Nix store or set sops.validateSopsFiles to false";
}])
secret.sopsFiles)
cfg.secrets)
);

systemd.user.services.sops-nix = lib.mkIf pkgs.stdenv.hostPlatform.isLinux {
Expand Down
69 changes: 48 additions & 21 deletions modules/sops/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,18 @@ let
users = config.users.users;
sops-install-secrets = cfg.package;
sops-install-secrets-check = cfg.validationPackage;
regularSecrets = lib.filterAttrs (_: v: !v.neededForUsers) cfg.secrets;
secretsForUsers = lib.filterAttrs (_: v: v.neededForUsers) cfg.secrets;
secretType = types.submodule ({ config, ... }: {
config = {
sopsFile = lib.mkOptionDefault cfg.defaultSopsFile;
sopsFileHash = mkOptionDefault (optionalString cfg.validateSopsFiles "${builtins.hashFile "sha256" config.sopsFile}");
};
secrets = mapAttrs (_: secret: removeAttrs secret ["sopsFile"]) cfg.secrets;
regularSecrets = lib.filterAttrs (_: v: !v.neededForUsers) secrets;
secretsForUsers = lib.filterAttrs (_: v: v.neededForUsers) secrets;
secretType = types.submodule ({ config, options, ... }: {
config = mkMerge [{
sopsFile = mkOptionDefault cfg.defaultSopsFile;
sopsFiles = mkIf (length cfg.defaultSopsFiles > 0) (mkOptionDefault cfg.defaultSopsFiles);
sopsFilesHash = mkOptionDefault (optionals cfg.validateSopsFiles (forEach config.sopsFiles (builtins.hashFile "sha256")));
}
{
sopsFiles = mkIf (config.sopsFile != null) (mkOverride options.sopsFile.highestPrio (mkBefore [config.sopsFile]));
}];
options = {
name = mkOption {
type = types.str;
Expand Down Expand Up @@ -71,17 +76,24 @@ let
'';
};
sopsFile = mkOption {
type = types.path;
type = types.nullOr types.path;
defaultText = "\${config.sops.defaultSopsFile}";
description = ''
Sops file the secret is loaded from.
'';
};
sopsFileHash = mkOption {
type = types.str;
sopsFiles = mkOption {
type = types.nonEmptyListOf types.path;
defaultText = "\${config.sops.defaultSopsFiles}";
description = ''
Sops files the secret is loaded from.
'';
};
sopsFilesHash = mkOption {
type = types.nonEmptyListOf types.str;
readOnly = true;
description = ''
Hash of the sops file, useful in <xref linkend="opt-systemd.services._name_.restartTriggers" />.
Hash of the sops files, useful in <xref linkend="opt-systemd.services._name_.restartTriggers" />.
'';
};
restartUnits = mkOption {
Expand Down Expand Up @@ -167,12 +179,21 @@ in {
};

defaultSopsFile = mkOption {
type = types.path;
type = types.nullOr types.path;
default = null;
description = ''
Default sops file used for all secrets.
'';
};
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And this one with a warning.


defaultSopsFiles = mkOption {
type = types.listOf types.path;
default = [];
description = ''
Default sops files used for all secrets.
'';
};

defaultSopsFormat = mkOption {
type = types.str;
default = "yaml";
Expand Down Expand Up @@ -331,17 +352,23 @@ in {
assertion = (filterAttrs (_: v: v.owner != "root" || v.group != "root") secretsForUsers) == {};
message = "neededForUsers cannot be used for secrets that are not root-owned";
}] ++ optionals cfg.validateSopsFiles (
concatLists (mapAttrsToList (name: secret: [{
assertion = builtins.pathExists secret.sopsFile;
message = "Cannot find path '${secret.sopsFile}' set in sops.secrets.${strings.escapeNixIdentifier name}.sopsFile";
} {
assertion =
builtins.isPath secret.sopsFile ||
(builtins.isString secret.sopsFile && hasPrefix builtins.storeDir secret.sopsFile);
message = "'${secret.sopsFile}' is not in the Nix store. Either add it to the Nix store or set sops.validateSopsFiles to false";
}]) cfg.secrets)
concatLists (mapAttrsToList
(name: secret:
concatMap
(sopsFile: [{
assertion = builtins.pathExists sopsFile;
message = "Cannot find path '${sopsFile}' set in sops.secrets.${strings.escapeNixIdentifier name}.sopsFiles";
} {
assertion =
builtins.isPath sopsFile ||
(builtins.isString sopsFile && hasPrefix builtins.storeDir sopsFile);
message = "'${sopsFile}' is not in the Nix store. Either add it to the Nix store or set sops.validateSopsFiles to false";
}])
secret.sopsFiles)
cfg.secrets)
);


sops.environment.SOPS_GPG_EXEC = mkIf (cfg.gnupg.home != null) (mkDefault "${pkgs.gnupg}/bin/gpg");

system.activationScripts = {
Expand Down
115 changes: 68 additions & 47 deletions pkgs/sops-install-secrets/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ type secret struct {
Path string `json:"path"`
Owner string `json:"owner"`
Group string `json:"group"`
SopsFile string `json:"sopsFile"`
SopsFiles []string `json:"sopsFiles"`
Format FormatType `json:"format"`
Mode string `json:"mode"`
RestartUnits []string `json:"restartUnits"`
Expand Down Expand Up @@ -258,40 +258,47 @@ func recurseSecretKey(keys map[string]interface{}, wantedKey string) (string, er
}

func decryptSecret(s *secret, sourceFiles map[string]plainData) error {
sourceFile := sourceFiles[s.SopsFile]
if sourceFile.data == nil || sourceFile.binary == nil {
plain, err := decrypt.File(s.SopsFile, string(s.Format))
if err != nil {
return fmt.Errorf("Failed to decrypt '%s': %w", s.SopsFile, err)
}
for i := len(s.SopsFiles) - 1; i >= 0; i-- {
sourceFile := sourceFiles[s.SopsFiles[i]]
if sourceFile.data == nil || sourceFile.binary == nil {
plain, err := decrypt.File(s.SopsFiles[i], string(s.Format))
if err != nil {
return fmt.Errorf("Failed to decrypt '%s': %w", s.SopsFiles[i], err)
}

switch s.Format {
case Binary, Dotenv, Ini:
sourceFile.binary = plain
case Yaml:
if err := yaml.Unmarshal(plain, &sourceFile.data); err != nil {
return fmt.Errorf("Cannot parse yaml of '%s': %w", s.SopsFiles[i], err)
}
case Json:
if err := json.Unmarshal(plain, &sourceFile.data); err != nil {
return fmt.Errorf("Cannot parse json of '%s': %w", s.SopsFiles[i], err)
}
default:
return fmt.Errorf("Secret of type %s in %s is not supported", s.Format, s.SopsFiles[i])
}
}
switch s.Format {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Assuming all sopsFiles have the same format

case Binary, Dotenv, Ini:
sourceFile.binary = plain
case Yaml:
if err := yaml.Unmarshal(plain, &sourceFile.data); err != nil {
return fmt.Errorf("Cannot parse yaml of '%s': %w", s.SopsFile, err)
}
case Json:
if err := json.Unmarshal(plain, &sourceFile.data); err != nil {
return fmt.Errorf("Cannot parse json of '%s': %w", s.SopsFile, err)
s.value = sourceFile.binary
case Yaml, Json:
strVal, err := recurseSecretKey(sourceFile.data, s.Key)
if err != nil {
continue
}
default:
return fmt.Errorf("Secret of type %s in %s is not supported", s.Format, s.SopsFile)
}
}
switch s.Format {
case Binary, Dotenv, Ini:
s.value = sourceFile.binary
case Yaml, Json:
strVal, err := recurseSecretKey(sourceFile.data, s.Key)
if err != nil {
return fmt.Errorf("secret %s in %s is not valid: %w", s.Name, s.SopsFile, err)
s.value = []byte(strVal)
}
s.value = []byte(strVal)

sourceFiles[s.SopsFiles[i]] = sourceFile
// Secret found
return nil
}
sourceFiles[s.SopsFile] = sourceFile
return nil

// Secret not found in any of the SopsFiles
return fmt.Errorf("secret %s in %v is not valid", s.Name, s.SopsFiles)
}

func decryptSecrets(secrets []secret) error {
Expand Down Expand Up @@ -395,40 +402,40 @@ func lookupKeysGroup() (int, error) {
return 0, fmt.Errorf("Can't find group 'keys' nor 'nogroup' (%w).", err2)
}

func (app *appContext) loadSopsFile(s *secret) (*secretFile, error) {
func (app *appContext) loadSopsFile(s *secret, sopsFile *string) (*secretFile, error) {
if app.checkMode == Manifest {
return &secretFile{firstSecret: s}, nil
}

cipherText, err := os.ReadFile(s.SopsFile)
cipherText, err := os.ReadFile(*sopsFile)
if err != nil {
return nil, fmt.Errorf("Failed reading %s: %w", s.SopsFile, err)
return nil, fmt.Errorf("Failed reading %s: %w", SopsFile, err)
}

var keys map[string]interface{}

switch s.Format {
case Binary:
if err := json.Unmarshal(cipherText, &keys); err != nil {
return nil, fmt.Errorf("Cannot parse json of '%s': %w", s.SopsFile, err)
return nil, fmt.Errorf("Cannot parse json of '%s': %w", *sopsFile, err)
}
return &secretFile{cipherText: cipherText, firstSecret: s}, nil
case Yaml:
if err := yaml.Unmarshal(cipherText, &keys); err != nil {
return nil, fmt.Errorf("Cannot parse yaml of '%s': %w", s.SopsFile, err)
return nil, fmt.Errorf("Cannot parse yaml of '%s': %w", *sopsFile, err)
}
case Dotenv:
env, err := godotenv.Unmarshal(string(cipherText))
if err != nil {
return nil, fmt.Errorf("Cannot parse dotenv of '%s': %w", s.SopsFile, err)
return nil, fmt.Errorf("Cannot parse dotenv of '%s': %w", *sopsFile, err)
}
keys = map[string]interface{}{}
for k, v := range env {
keys[k] = v
}
case Json:
if err := json.Unmarshal(cipherText, &keys); err != nil {
return nil, fmt.Errorf("Cannot parse json of '%s': %w", s.SopsFile, err)
return nil, fmt.Errorf("Cannot parse json of '%s': %w", *sopsFile, err)
}
}

Expand All @@ -441,14 +448,14 @@ func (app *appContext) loadSopsFile(s *secret) (*secretFile, error) {

func (app *appContext) validateSopsFile(s *secret, file *secretFile) error {
if file.firstSecret.Format != s.Format {
return fmt.Errorf("secret %s defined the format of %s as %s, but it was specified as %s in %s before",
s.Name, s.SopsFile, s.Format,
return fmt.Errorf("secret %s defined the format of %v as %s, but it was specified as %s in %s before",
s.Name, s.SopsFiles, s.Format,
file.firstSecret.Format, file.firstSecret.Name)
}
if app.checkMode != Manifest && (!(s.Format == Binary || s.Format == Dotenv || s.Format == Ini)) {
_, err := recurseSecretKey(file.keys, s.Key)
if err != nil {
return fmt.Errorf("secret %s in %s is not valid: %w", s.Name, s.SopsFile, err)
return fmt.Errorf("secret %s in %s is not valid: %v", s.Name, s.SopsFiles, err)
}
}
return nil
Expand Down Expand Up @@ -495,17 +502,31 @@ func (app *appContext) validateSecret(secret *secret) error {
return fmt.Errorf("Unsupported format %s for secret %s", secret.Format, secret.Name)
}

file, ok := app.secretFiles[secret.SopsFile]
if !ok {
maybeFile, err := app.loadSopsFile(secret)
if err != nil {
return err
files := []secretFile{}
for _, sopsFile := range secret.SopsFiles {
file, ok := app.secretFiles[sopsFile]
if !ok {
maybeFile, err := app.loadSopsFile(secret, &sopsFile)
if err != nil {
return err
}
app.secretFiles[sopsFile] = *maybeFile
file = *maybeFile
}
app.secretFiles[secret.SopsFile] = *maybeFile
file = *maybeFile
files = append(files, file)
}

return app.validateSopsFile(secret, &file)
for i := len(files) - 1; i >= 0; i-- {
err := app.validateSopsFile(secret, &files[i])
if err == nil {
// Found valid sopsFile
break
} else if i == 0 {
// No valid sopsFile found in sopsFiles
return fmt.Errorf("Failed to find valid secret %s in %v", secret.Name, secret.SopsFiles)
}
}
return nil
}

func (app *appContext) validateManifest() error {
Expand Down
Loading