initial commit

.github/workflows/pythonrelease.yml

@@ -0,0 +1,26 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install Poetry
+      run: |
+        pip install poetry
+    - name: Build and publish
+      env:
+        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
+      run: |
+        poetry version ${GITHUB_REF/refs\/tags\/v/}
+        poetry build
+
+        poetry publish

.github/workflows/pythontest.yml

@@ -0,0 +1,26 @@
+name: Tests
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install Poetry and Dependencies
+      run: |
+        pip install poetry
+        poetry install
+    - name: Lint with flake8
+      run: |
+        poetry run flake8 . --count --max-complexity=10 --max-line-length=120 --statistics
+      working-directory: ./src
+    - name: Static Typechecking with MyPy
+      run: |
+        poetry run mypy -p secrets_tool
+      working-directory: ./src

.gitignore

@@ -0,0 +1,4 @@
+.idea/
+**/.mypy_cache
+src/secrets_tool.egg-info
+dist/

README.md

@@ -0,0 +1,58 @@
+# Secrets Tool
+This is a small tool which helps to encrypt secrets that must be committed to a Git repository.
+
+It has the advantage to natively support partial encryption of YAML files. This is of great advantage, as it allows to see the YAML file structure even when some of its contents are encrypted (your PR reviewers and diff tools will thank you)
+
+## Prerequisites
+* Python >= 3.7
+* Having the following packages installed: `pip install ruamel.yaml cryptography`
+
+## Usage
+The tool reads a list of files to encrypt/decrypt from a `.gitignore` file. In there it will only consider files that are sorrounded by a comment block as in the following example:
+
+```
+# BEGIN ENCRYPTED
+kaas-rubik-stage/values.yaml
+# END ENCRYPTED
+```
+
+Run the tool by giving the `.gitignore` file as an argument, together with either a `encrypt` or `decrypt` command:
+
+```
+cd <REPOSITORY_ROOT>
+python -m utils.secrets_tool k8s_helm/.gitignore encrypt
+```
+
+## Syntax
+The tool provides different encryption handlers for all kind of file types.
+* `yaml` for YAML files that are used by tools which are okay having a `!decrypted` tag in front of strings
+* `yamlcompat` for tools that don't like the additional 'encryption marker' tag.
+* `generic` for all other file types. It encrypts the complete file.
+
+The desired encryption handler is inferred from the filetype - or it can be given explicitly in the gitignore file using the `# type:` hint:
+
+```
+# BEGIN ENCRYPTED
+kaas-rubik-stage/values.yaml
+
+# type: yaml
+kaas-rubik-stage/values2.txt
+# END ENCRYPTED
+```
+
+### yamlcompat
+This encryption handler can encrypt individual YAML keys without relying on 'parser visible' changes in the YAML file structure.
+Instead of marking the desired keys directly in the file, they are listed in the .gitignore file using a `# data: ` comment:
+
+```
+# BEGIN ENCRYPTED
+kaas-rubik-stage/values.yaml
+
+# type: yamlcompat
+# data: splunk.apiToken
+# data: splunk.host
+kaas-rubik-stage/values2.yaml
+# END ENCRYPTED
+```
+
+*WARNING* It is recommended to use the normal YAML handler whenever possible. When using the yamlcompat module, you split up your encryption logic over multiple files, which might lead to errors (especially on fragile YAML files that contain unnamed structures - like lists)

pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "secrets_tool"
+description = "A lightweight tool to easily encrypt/decrypt secrets inside a repository"
+authors = ["Alexander Hungenberg <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/defreng/secrets-tool"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+# Version is set by GitHub action workflow (on release)
+version = "0.0.1+local"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+cryptography = "^2.9.2"
+"ruamel.yaml" = "^0.16.10"
+
+[tool.poetry.scripts]
+secrets_tool = "secrets_tool.__main__:main"
+
+
+[tool.poetry.dev-dependencies]
+mypy = "^0.782"
+flake8 = "^3.8.3"
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

src/secrets_tool/__main__.py

@@ -0,0 +1,74 @@
+import argparse
+import re
+from pathlib import Path
+
+from .handlers.generic import GenericFileHandler
+from .handlers.yaml import YamlFileHandler
+from .handlers.yamlcompat import YamlCompatFileHandler
+
+
+HANDLERS = {
+    'yaml': YamlFileHandler,
+    'yamlcompat': YamlCompatFileHandler,
+    'generic': GenericFileHandler,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename')
+    parser.add_argument('command', choices=('e', 'encrypt', 'd', 'decrypt'))
+
+    args = parser.parse_args()
+
+    secrets_file_locations = (
+        Path(Path.home(), '.tap-rubik-key'),
+        Path(Path.home(), '.secrets-tool-key'),
+    )
+
+    for path in secrets_file_locations:
+        if path.exists():
+            key = path.read_bytes().strip()[:32]
+            break
+    else:
+        raise Exception('Could not find file with your secret key.')
+
+    gitignore_filepath = Path(args.filename)
+    base_path = gitignore_filepath.parent
+
+    gi_content = gitignore_filepath.read_text('ascii')
+    gi_content_match = re.search(r'(?s)# BEGIN ENCRYPTED\n(.*)\n# END ENCRYPTED', gi_content)
+
+    if gi_content_match is None:
+        raise Exception("Couldn't find a # BEGIN/END ENCRYPTED section in the provided .gitignore file")
+    gi_content_match = gi_content_match.group(1)
+
+    statement_expr = r'^(?:# type: (?P<type>\w+)\n(?P<data_raw>(?:# data: .+\n)*))?^(?P<filepath>[^#\n].*)$'
+    for statement in re.finditer(statement_expr, gi_content_match, flags=re.MULTILINE):
+        data_raw = statement.group('data_raw')
+        data = [raw.strip() for raw in data_raw.split('# data: ') if len(raw) > 0] if data_raw is not None else None
+
+        filepath = base_path / statement.group('filepath')
+
+        type_ = statement.group('type')
+        if type_ is None:
+            if filepath.suffix in ('.yml', '.yaml'):
+                type_ = 'yaml'
+            else:
+                type_ = 'generic'
+        handler = HANDLERS[type_]
+
+        if args.command in ('e', 'encrypt'):
+            target_path = Path(str(filepath) + '.enc')
+
+            handler(filepath, data).dump_encrypted(target_path, key)
+            print(f'ENCRYPTED {filepath.resolve()} (into {target_path.resolve()})')
+        elif args.command in ('d', 'decrypt'):
+            source_path = Path(str(filepath) + '.enc')
+
+            handler(source_path, data).dump_decrypted(filepath, key)
+            print(f'DECRYPTED {source_path.resolve()} (into {filepath.resolve()})')
+
+
+if __name__ == '__main__':
+    main()

src/secrets_tool/cipher.py

@@ -0,0 +1,56 @@
+"""
+Implementing custom encryption, as the default "fernet" method from the cryptography package
+uses random initialization numbers, which cause a different encrypted string for identical content all the time.
+
+We don't need message authentication in our case, and changing strings would be annoying in Git.
+"""
+import base64
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, padding
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+
+def encrypt(data: bytes, key: bytes, iv_bytes: bytes) -> bytes:
+    """
+    data can be bytes of arbitrary length
+    key must be 32 bytes long
+    iv_bytes can be of arbitrary length, as they will be hashed
+    """
+    iv_digest = hashes.Hash(hashes.SHA256(), default_backend())
+    iv_digest.update(iv_bytes)
+    iv = iv_digest.finalize()[:16]
+
+    algorithm = algorithms.AES(key)
+    mode = modes.CBC(iv)
+
+    cipher = Cipher(algorithm, mode=mode, backend=default_backend())
+    encryptor = cipher.encryptor()
+
+    padder = padding.PKCS7(algorithm.block_size).padder()
+    to_encrypt = padder.update(iv + data) + padder.finalize()
+
+    encrypted = encryptor.update(to_encrypt) + encryptor.finalize()
+    return base64.b64encode(encrypted)
+
+
+def decrypt(data: bytes, key: bytes) -> bytes:
+    """
+    data should be encrypted binary, encoded as base64 - as it is produced by 'encrypt()'
+    key must be 32 bytes long
+    """
+
+    data = base64.b64decode(data)
+    iv = data[:16]
+    message = data[16:]
+
+    algorithm = algorithms.AES(key)
+    mode = modes.CBC(iv)
+
+    cipher = Cipher(algorithm, mode=mode, backend=default_backend())
+    decryptor = cipher.decryptor()
+    decrypted_padded = decryptor.update(message) + decryptor.finalize()
+
+    unpadder = padding.PKCS7(algorithm.block_size).unpadder()
+    decrypted = unpadder.update(decrypted_padded) + unpadder.finalize()
+    return decrypted

src/secrets_tool/handlers/generic.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+from typing import List
+
+from .. import cipher
+
+
+class GenericFileHandler:
+    def __init__(self, filepath: Path, data: List[str]):
+        self.filepath = filepath
+
+    def dump_decrypted(self, target: Path, key: bytes):
+        target.write_bytes(cipher.decrypt(self.filepath.read_bytes(), key))
+
+    def dump_encrypted(self, target: Path, key: bytes):
+        target.write_bytes(cipher.encrypt(self.filepath.read_bytes(), key, str(self.filepath).encode('utf8')))

src/secrets_tool/handlers/yaml.py

@@ -0,0 +1,91 @@
+"""
+Classes to encrypt and decrypt YAML files
+
+- Encryption operation will look for fields with the "!decrypted" hint and replace them with an encrypted version of
+  their content. This will be marked with the "!encrypted" hint
+- Decryption operation will look for fields with the "!encrypted" hint and replace them with a decrypted version of
+  their content. This will be marked with the "!decrypted" hint
+"""
+from pathlib import Path
+from typing import List
+
+import ruamel.yaml
+
+from .. import cipher
+
+
+class DecryptedString:
+    yaml_tag = '!decrypted'
+
+    def __init__(self, data: str):
+        self.data = data
+
+    @classmethod
+    def from_encrypted(cls, data: str, key):
+        return cls(cipher.decrypt(data.encode('ascii'), key).decode('utf8'))
+
+    @classmethod
+    def from_yaml(cls, constructor, node):
+        return cls(node.value)
+
+    @classmethod
+    def to_yaml(cls, representer, node):
+        return representer.represent_scalar(cls.yaml_tag, node.data)
+
+
+class EncryptedString:
+    yaml_tag = '!encrypted'
+
+    def __init__(self, data: str):
+        self.data = data
+
+    @classmethod
+    def from_decrypted(cls, data: str, key: bytes, iv: bytes):
+        return cls(cipher.encrypt(data.encode('utf8'), key, iv).decode('ascii'))
+
+    @classmethod
+    def from_yaml(cls, constructor, node):
+        return cls(node.value)
+
+    @classmethod
+    def to_yaml(cls, representer, node):
+        return representer.represent_scalar(cls.yaml_tag, node.data)
+
+
+class YamlFileHandler:
+    def __init__(self, filepath: Path, data: List[str]):
+        self.yaml = ruamel.yaml.YAML()
+        self.yaml.register_class(DecryptedString)
+        self.yaml.register_class(EncryptedString)
+
+        self.filepath = filepath
+
+    def dump_decrypted(self, target: Path, key: bytes):
+        tree = self.yaml.load(self.filepath)
+
+        self._walk_item(tree, EncryptedString,
+                        lambda enc_string, iv: DecryptedString.from_encrypted(enc_string.data, key))
+
+        self.yaml.dump(tree, target)
+
+    def dump_encrypted(self, target: Path, key: bytes):
+        tree = self.yaml.load(self.filepath)
+
+        self._walk_item(tree, DecryptedString,
+                        lambda dec_string, iv: EncryptedString.from_decrypted(dec_string.data, key, iv.encode('utf8')))
+
+        self.yaml.dump(tree, target)
+
+    def _walk_item(self, item, type_, callback, path=''):
+        if isinstance(item, dict):
+            for key in item.keys():
+                if isinstance(item[key], type_):
+                    item[key] = callback(item[key], path + f'.{key}')
+                else:
+                    self._walk_item(item[key], type_, callback, path=path + f'.{key}')
+        elif isinstance(item, list):
+            for i in range(len(item)):
+                if isinstance(item[i], type_):
+                    item[i] = callback(item[i], path + f'.{i}')
+                else:
+                    self._walk_item(item[i], type_, callback, path=path + f'.{i}')