From e3893679841270d5a3aa2a585d3b05c568f6dc61 Mon Sep 17 00:00:00 2001 From: DerTiedemann Date: Mon, 11 Nov 2024 10:08:45 +0100 Subject: [PATCH 1/2] feat: initial version of gsm secret loader (#213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is the basis for the workload identity integration for GH workflows. Based on context it will load all secrets that are specified, transform their names to screaming snake case and export them. There are some tests and one is failing and i dont know why. All basic usecases work though. --------- Co-authored-by: Yannick Röder <33963579+yannick-roeder@users.noreply.github.com> --- .github/workflows/_test-python-actions.yaml | 22 +++ .gitignore | 2 + actions/gcp-gsm-load-secrets/README.md | 1 + actions/gcp-gsm-load-secrets/action.yaml | 42 ++++++ actions/gcp-gsm-parse-secrets/Dockerfile | 26 ++++ actions/gcp-gsm-parse-secrets/README.md | 1 + actions/gcp-gsm-parse-secrets/action.yaml | 15 ++ actions/gcp-gsm-parse-secrets/main.py | 66 +++++++++ actions/gcp-gsm-parse-secrets/poetry.lock | 138 +++++++++++++++++++ actions/gcp-gsm-parse-secrets/pyproject.toml | 15 ++ actions/gcp-gsm-parse-secrets/tests.py | 18 +++ docs/actions/gcp-gsm-load-secrets/README.md | 56 ++++++++ docs/actions/gcp-gsm-parse-secrets/README.md | 51 +++++++ 13 files changed, 453 insertions(+) create mode 100644 .github/workflows/_test-python-actions.yaml create mode 120000 actions/gcp-gsm-load-secrets/README.md create mode 100644 actions/gcp-gsm-load-secrets/action.yaml create mode 100644 actions/gcp-gsm-parse-secrets/Dockerfile create mode 120000 actions/gcp-gsm-parse-secrets/README.md create mode 100644 actions/gcp-gsm-parse-secrets/action.yaml create mode 100644 actions/gcp-gsm-parse-secrets/main.py create mode 100644 actions/gcp-gsm-parse-secrets/poetry.lock create mode 100644 actions/gcp-gsm-parse-secrets/pyproject.toml create mode 100644 actions/gcp-gsm-parse-secrets/tests.py create mode 100644 docs/actions/gcp-gsm-load-secrets/README.md create mode 100644 docs/actions/gcp-gsm-parse-secrets/README.md diff --git a/.github/workflows/_test-python-actions.yaml b/.github/workflows/_test-python-actions.yaml new file mode 100644 index 000000000..c29974bf8 --- /dev/null +++ b/.github/workflows/_test-python-actions.yaml @@ -0,0 +1,22 @@ +name: Test python-setup-poetry action + +on: + pull_request: + branches: + - main + paths: + - actions/parse-secret-definitions + +jobs: + tests: + name: Python action unit tests + runs-on: ubuntu-latest + steps: + - uses: bakdata/ci-templates/actions/checkout@1.32.0 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.12" + - name: Run all tests + run: | + find . -name "tests.py" -print0 | xargs -0i python -m unittest diff --git a/.gitignore b/.gitignore index d01517ef2..4c713b125 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ tmp* auto-doc* ./test* +**/__pycache__/ +**/venv/ diff --git a/actions/gcp-gsm-load-secrets/README.md b/actions/gcp-gsm-load-secrets/README.md new file mode 120000 index 000000000..fad2cd6ee --- /dev/null +++ b/actions/gcp-gsm-load-secrets/README.md @@ -0,0 +1 @@ +docs/actions/gcp-gsm-load-secrets/README.md \ No newline at end of file diff --git a/actions/gcp-gsm-load-secrets/action.yaml b/actions/gcp-gsm-load-secrets/action.yaml new file mode 100644 index 000000000..648232c9f --- /dev/null +++ b/actions/gcp-gsm-load-secrets/action.yaml @@ -0,0 +1,42 @@ +name: "Load secrets from Google Secret Manager" +description: "Load secrets from Google Secret Manager and inject them into the environment" +inputs: + gke-service-account: + description: "GKE service account for authentication" + required: true + gke-project-name: + description: "GKE project name for authentication" + required: true + workload-identity-provider: + description: "Workload identity provider for authentication" + required: true + secrets-to-inject: + description: "Secrets to inject into the environment" + required: true + export-to-environment: + description: "Export secrets to environment" + required: false + default: true +outputs: + secrets: + description: "Secrets loaded from Secret Manager" + value: ${{ steps.secrets.outputs.secrets }} +runs: + using: "composite" + steps: + - name: Authenticate at GCloud + uses: "google-github-actions/auth@v2" + with: + project_id: ${{ inputs.gke-project-name }} + workload_identity_provider: ${{ inputs.workload-identity-provider }} + service_account: ${{ inputs.gke-service-account }} + - id: "parse_secrets" + uses: "bakdata/ci-templates/actions/parse-secrets-definitions@1.48.0" + with: + project_name: ${{ inputs.gke-project-name }} + secrets_list: ${{ inputs.secrets-to-inject }} + - id: "secrets" + uses: "google-github-actions/get-secretmanager-secrets@v2" + with: + secrets: ${{ steps.parse_secrets.outputs.secrets-list }} + export_to_environment: ${{ inputs.export-to-environment }} diff --git a/actions/gcp-gsm-parse-secrets/Dockerfile b/actions/gcp-gsm-parse-secrets/Dockerfile new file mode 100644 index 000000000..7694a30e7 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3-slim AS builder +RUN pip install poetry==1.8.2 + +ENV POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_VIRTUALENVS_CREATE=1 \ + POETRY_CACHE_DIR=/tmp/poetry_cache + +WORKDIR /app +COPY pyproject.toml poetry.lock ./ +COPY main.py ./ +RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR + +# A distroless container image with Python and some basics like SSL certificates +# https://github.com/GoogleContainerTools/dis/i/itroless +FROM gcr.io/distroless/python3-debian12 + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /app /app +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +WORKDIR /app +ENV PYTHONPATH /app +CMD ["/app/main.py"] \ No newline at end of file diff --git a/actions/gcp-gsm-parse-secrets/README.md b/actions/gcp-gsm-parse-secrets/README.md new file mode 120000 index 000000000..9567d8ebe --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/README.md @@ -0,0 +1 @@ +docs/actions/gcp-gsm-parse-secrets/README.md \ No newline at end of file diff --git a/actions/gcp-gsm-parse-secrets/action.yaml b/actions/gcp-gsm-parse-secrets/action.yaml new file mode 100644 index 000000000..129f26b04 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/action.yaml @@ -0,0 +1,15 @@ +name: "Parse secrets from GSM" +description: "Transform secrets into a common format" +inputs: + secrets-list: + description: "Secrets to inject into the environment" + required: true + project-name: + description: "GKE project name where the secrets are stored" + required: true +outputs: + secrets-list: + description: "secret list with correct format" +runs: + using: "docker" + image: "Dockerfile" diff --git a/actions/gcp-gsm-parse-secrets/main.py b/actions/gcp-gsm-parse-secrets/main.py new file mode 100644 index 000000000..b67015c8f --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/main.py @@ -0,0 +1,66 @@ +import os +import re + +import typer +from typing_extensions import Annotated + +DEFAULT_DELIMITER = "!!!" + +# CAVEAT: will only work for one project at a time +# to add secrets form another project, invoke the action a second time with the other project name + + +# Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here: +# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter +def set_github_action_output(output_name, output_value, delim): + if os.environ.get("GITHUB_ACTION"): + f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a") + f.write( + f"{output_name}<<{delim}\n{output_value}{delim}\n" + ) # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work + f.close() + else: + print("would have set output", output_name, "to", output_value) + + +# removes special characters and replace with underscores, successive special characters are replaced with a single underscore +# convert to uppercase +# if the secret would end in an underscore, remove it +# format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION +def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER): + if delim in secret: + raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword") + components = secret.split("/") + + if len(components) > 2: + raise ValueError( + f"Invalid secret definition: {secret}, not in the format 'secret_name/'" + ) + secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_") + if secret_name == "": + raise ValueError( + f"Invalid secret definition: {components[0]} is not a valid secret name" + ) + out = f"{secret_name}:{project_name}/{components[0]}" + if len(components) == 2 and len(components[1]) != 0: + out += f"/{components[1]}" + return out + + +def main( + input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")], + gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")], + github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER, +): + # Deduplicate the input secrets + input_secrets = set(input_secrets.splitlines()) + + output = "" + for secret in input_secrets: + output += parse_secret(secret, gcp_project, github_output_delimter) + "\n" + + set_github_action_output("secrets-list", output, github_output_delimter) + + +if __name__ == "__main__": + typer.run(main) diff --git a/actions/gcp-gsm-parse-secrets/poetry.lock b/actions/gcp-gsm-parse-secrets/poetry.lock new file mode 100644 index 000000000..f09a27ab2 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/poetry.lock @@ -0,0 +1,138 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "rich" +version = "13.9.3" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "rich-13.9.3-py3-none-any.whl", hash = "sha256:9836f5096eb2172c9e77df411c1b009bace4193d6a481d534fea75ebba758283"}, + {file = "rich-13.9.3.tar.gz", hash = "sha256:bc1e01b899537598cf02579d2b9f4a415104d3fc439313a7a2c165d76557a08e"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + +[[package]] +name = "typer" +version = "0.12.5" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +files = [ + {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, + {file = "typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "a8b347ff4d818c262736c1211b31d62c29e58b723863a9213ab5041ac1d54b74" diff --git a/actions/gcp-gsm-parse-secrets/pyproject.toml b/actions/gcp-gsm-parse-secrets/pyproject.toml new file mode 100644 index 000000000..3bf8e5a93 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "gcp-gsm-parse-secrets" +version = "0.1.0" +description = "" +authors = ["Jan Max Tiedemann "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +typer = "^0.12.5" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/actions/gcp-gsm-parse-secrets/tests.py b/actions/gcp-gsm-parse-secrets/tests.py new file mode 100644 index 000000000..66fee5679 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/tests.py @@ -0,0 +1,18 @@ +import unittest + +from main import parse_secret + +class TestParseSecret(unittest.TestCase): + def test_parse_secret(self): + self.assertEqual(parse_secret("secret_name", "project_name"), "SECRET_NAME:project_name/secret_name") + self.assertEqual(parse_secret("secret_name/version", "project_name"), "SECRET_NAME:project_name/secret_name/version") + self.assertEqual(parse_secret("123-456", "project_name"), "123_456:project_name/123-456") + self.assertEqual(parse_secret("123___123___123", "project_name"), "123_123_123:project_name/123___123___123") + self.assertEqual(parse_secret("i-like_trains__why_this?", "project_name"), "I_LIKE_TRAINS_WHY_THIS:project_name/i-like_trains__why_this?") + + def test_parse_secret_special(self): + # FIXME: this test is failing and i dont know why + self.assertEqual(parse_secret("123&&123()123__123*__*_123", "project_name"), "123_123_123_123:project_name/123&&123()123__123*__*_123") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/docs/actions/gcp-gsm-load-secrets/README.md b/docs/actions/gcp-gsm-load-secrets/README.md new file mode 100644 index 000000000..0d87d5967 --- /dev/null +++ b/docs/actions/gcp-gsm-load-secrets/README.md @@ -0,0 +1,56 @@ +# gcp-gsm-load-secrets + +This action is set to replace GitHub actions integrated secret management. + +## Usage + +To load a secret from GSM figure out the following: + +- check if the repository has access to the secret + - repository is owned by bakdata + - repository is private + - even if the labels are correctly set, you will need to run Terraform to set the proper roles +- use this template: + +```yaml +- name: Load secrets + id: load-secrets + uses: bakdata/ci-templates/actions/gcp-gsm-load-secrets + with: + gke-project-name: + gke-project-id: + secrets-to-inject: |- + / + / +``` + +- it is possible to load multiple secrets in the same call +- loaded secrets will be injected as environment variables and the name will be cannonicalized to SCREAMING_SNAKE_CASE. Example: `i-like_trains__why_this?` -> `I_LIKE_TRAINS_WHY_THIS` + +## References + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| -------------------------- | ------ | -------- | -------- | --------------------------------------------- | +| export-to-environment | string | false | `"true"` | Export secrets to environment | +| gke-project-name | string | true | | GKE project name for authentication | +| gke-service-account | string | true | | GKE service account for authentication | +| secrets-to-inject | string | true | | Secrets to inject into the environment | +| workload-identity-provider | string | true | | Workload identity provider for authentication | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------- | ------ | ---------------------------------- | +| secrets | string | Secrets loaded from Secret Manager | + + + +### Secrets diff --git a/docs/actions/gcp-gsm-parse-secrets/README.md b/docs/actions/gcp-gsm-parse-secrets/README.md new file mode 100644 index 000000000..1516f958e --- /dev/null +++ b/docs/actions/gcp-gsm-parse-secrets/README.md @@ -0,0 +1,51 @@ +# gcp-gsm-parse-secrets + +Converts a lists of strings of secrets references into screaming snake case. Look at the tests.py for furhter details. + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| ------------ | ------ | -------- | ------- | --------------------------------------------- | +| project-name | string | true | | GKE project name where the secrets are stored | +| secrets-list | string | true | | Secrets to inject into the environment | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------------ | ------ | ------------------------------- | +| secrets-list | string | secret list with correct format | + + + +### Secrets + +## References + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| ------------ | ------ | -------- | ------- | --------------------------------------------- | +| project-name | string | true | | GKE project name where the secrets are stored | +| secrets-list | string | true | | Secrets to inject into the environment | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------------ | ------ | ------------------------------- | +| secrets-list | string | secret list with correct format | + + + +### Secrets From 6c4d588e43aedf01bf3919fb0ec7f037ecdb0c2d Mon Sep 17 00:00:00 2001 From: bakdata-bot Date: Mon, 11 Nov 2024 09:09:46 +0000 Subject: [PATCH 2/2] =?UTF-8?q?Bump=20version=201.47.0=20=E2=86=92=201.48.?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8158fd352..180cf2538 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,2 +1,2 @@ [bumpversion] -current_version = 1.47.0 +current_version = 1.48.0