diff --git a/.github/workflows/_test-python-actions.yaml b/.github/workflows/_test-python-actions.yaml new file mode 100644 index 000000000..c29974bf8 --- /dev/null +++ b/.github/workflows/_test-python-actions.yaml @@ -0,0 +1,22 @@ +name: Test python-setup-poetry action + +on: + pull_request: + branches: + - main + paths: + - actions/parse-secret-definitions + +jobs: + tests: + name: Python action unit tests + runs-on: ubuntu-latest + steps: + - uses: bakdata/ci-templates/actions/checkout@1.32.0 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.12" + - name: Run all tests + run: | + find . -name "tests.py" -print0 | xargs -0i python -m unittest diff --git a/.gitignore b/.gitignore index d01517ef2..4c713b125 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ tmp* auto-doc* ./test* +**/__pycache__/ +**/venv/ diff --git a/actions/gcp-gsm-load-secrets/README.md b/actions/gcp-gsm-load-secrets/README.md new file mode 120000 index 000000000..fad2cd6ee --- /dev/null +++ b/actions/gcp-gsm-load-secrets/README.md @@ -0,0 +1 @@ +docs/actions/gcp-gsm-load-secrets/README.md \ No newline at end of file diff --git a/actions/gcp-gsm-load-secrets/action.yaml b/actions/gcp-gsm-load-secrets/action.yaml new file mode 100644 index 000000000..648232c9f --- /dev/null +++ b/actions/gcp-gsm-load-secrets/action.yaml @@ -0,0 +1,42 @@ +name: "Load secrets from Google Secret Manager" +description: "Load secrets from Google Secret Manager and inject them into the environment" +inputs: + gke-service-account: + description: "GKE service account for authentication" + required: true + gke-project-name: + description: "GKE project name for authentication" + required: true + workload-identity-provider: + description: "Workload identity provider for authentication" + required: true + secrets-to-inject: + description: "Secrets to inject into the environment" + required: true + export-to-environment: + description: "Export secrets to environment" + required: false + default: true +outputs: + secrets: + description: "Secrets loaded from Secret Manager" + value: ${{ steps.secrets.outputs.secrets }} +runs: + using: "composite" + steps: + - name: Authenticate at GCloud + uses: "google-github-actions/auth@v2" + with: + project_id: ${{ inputs.gke-project-name }} + workload_identity_provider: ${{ inputs.workload-identity-provider }} + service_account: ${{ inputs.gke-service-account }} + - id: "parse_secrets" + uses: "bakdata/ci-templates/actions/parse-secrets-definitions@1.48.0" + with: + project_name: ${{ inputs.gke-project-name }} + secrets_list: ${{ inputs.secrets-to-inject }} + - id: "secrets" + uses: "google-github-actions/get-secretmanager-secrets@v2" + with: + secrets: ${{ steps.parse_secrets.outputs.secrets-list }} + export_to_environment: ${{ inputs.export-to-environment }} diff --git a/actions/gcp-gsm-parse-secrets/Dockerfile b/actions/gcp-gsm-parse-secrets/Dockerfile new file mode 100644 index 000000000..7694a30e7 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3-slim AS builder +RUN pip install poetry==1.8.2 + +ENV POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_VIRTUALENVS_CREATE=1 \ + POETRY_CACHE_DIR=/tmp/poetry_cache + +WORKDIR /app +COPY pyproject.toml poetry.lock ./ +COPY main.py ./ +RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR + +# A distroless container image with Python and some basics like SSL certificates +# https://github.com/GoogleContainerTools/dis/i/itroless +FROM gcr.io/distroless/python3-debian12 + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /app /app +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +WORKDIR /app +ENV PYTHONPATH /app +CMD ["/app/main.py"] \ No newline at end of file diff --git a/actions/gcp-gsm-parse-secrets/README.md b/actions/gcp-gsm-parse-secrets/README.md new file mode 120000 index 000000000..9567d8ebe --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/README.md @@ -0,0 +1 @@ +docs/actions/gcp-gsm-parse-secrets/README.md \ No newline at end of file diff --git a/actions/gcp-gsm-parse-secrets/action.yaml b/actions/gcp-gsm-parse-secrets/action.yaml new file mode 100644 index 000000000..129f26b04 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/action.yaml @@ -0,0 +1,15 @@ +name: "Parse secrets from GSM" +description: "Transform secrets into a common format" +inputs: + secrets-list: + description: "Secrets to inject into the environment" + required: true + project-name: + description: "GKE project name where the secrets are stored" + required: true +outputs: + secrets-list: + description: "secret list with correct format" +runs: + using: "docker" + image: "Dockerfile" diff --git a/actions/gcp-gsm-parse-secrets/main.py b/actions/gcp-gsm-parse-secrets/main.py new file mode 100644 index 000000000..b67015c8f --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/main.py @@ -0,0 +1,66 @@ +import os +import re + +import typer +from typing_extensions import Annotated + +DEFAULT_DELIMITER = "!!!" + +# CAVEAT: will only work for one project at a time +# to add secrets form another project, invoke the action a second time with the other project name + + +# Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here: +# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter +def set_github_action_output(output_name, output_value, delim): + if os.environ.get("GITHUB_ACTION"): + f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a") + f.write( + f"{output_name}<<{delim}\n{output_value}{delim}\n" + ) # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work + f.close() + else: + print("would have set output", output_name, "to", output_value) + + +# removes special characters and replace with underscores, successive special characters are replaced with a single underscore +# convert to uppercase +# if the secret would end in an underscore, remove it +# format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION +def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER): + if delim in secret: + raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword") + components = secret.split("/") + + if len(components) > 2: + raise ValueError( + f"Invalid secret definition: {secret}, not in the format 'secret_name/'" + ) + secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_") + if secret_name == "": + raise ValueError( + f"Invalid secret definition: {components[0]} is not a valid secret name" + ) + out = f"{secret_name}:{project_name}/{components[0]}" + if len(components) == 2 and len(components[1]) != 0: + out += f"/{components[1]}" + return out + + +def main( + input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")], + gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")], + github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER, +): + # Deduplicate the input secrets + input_secrets = set(input_secrets.splitlines()) + + output = "" + for secret in input_secrets: + output += parse_secret(secret, gcp_project, github_output_delimter) + "\n" + + set_github_action_output("secrets-list", output, github_output_delimter) + + +if __name__ == "__main__": + typer.run(main) diff --git a/actions/gcp-gsm-parse-secrets/poetry.lock b/actions/gcp-gsm-parse-secrets/poetry.lock new file mode 100644 index 000000000..f09a27ab2 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/poetry.lock @@ -0,0 +1,138 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "rich" +version = "13.9.3" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "rich-13.9.3-py3-none-any.whl", hash = "sha256:9836f5096eb2172c9e77df411c1b009bace4193d6a481d534fea75ebba758283"}, + {file = "rich-13.9.3.tar.gz", hash = "sha256:bc1e01b899537598cf02579d2b9f4a415104d3fc439313a7a2c165d76557a08e"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + +[[package]] +name = "typer" +version = "0.12.5" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +files = [ + {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, + {file = "typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "a8b347ff4d818c262736c1211b31d62c29e58b723863a9213ab5041ac1d54b74" diff --git a/actions/gcp-gsm-parse-secrets/pyproject.toml b/actions/gcp-gsm-parse-secrets/pyproject.toml new file mode 100644 index 000000000..3bf8e5a93 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "gcp-gsm-parse-secrets" +version = "0.1.0" +description = "" +authors = ["Jan Max Tiedemann "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +typer = "^0.12.5" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/actions/gcp-gsm-parse-secrets/tests.py b/actions/gcp-gsm-parse-secrets/tests.py new file mode 100644 index 000000000..66fee5679 --- /dev/null +++ b/actions/gcp-gsm-parse-secrets/tests.py @@ -0,0 +1,18 @@ +import unittest + +from main import parse_secret + +class TestParseSecret(unittest.TestCase): + def test_parse_secret(self): + self.assertEqual(parse_secret("secret_name", "project_name"), "SECRET_NAME:project_name/secret_name") + self.assertEqual(parse_secret("secret_name/version", "project_name"), "SECRET_NAME:project_name/secret_name/version") + self.assertEqual(parse_secret("123-456", "project_name"), "123_456:project_name/123-456") + self.assertEqual(parse_secret("123___123___123", "project_name"), "123_123_123:project_name/123___123___123") + self.assertEqual(parse_secret("i-like_trains__why_this?", "project_name"), "I_LIKE_TRAINS_WHY_THIS:project_name/i-like_trains__why_this?") + + def test_parse_secret_special(self): + # FIXME: this test is failing and i dont know why + self.assertEqual(parse_secret("123&&123()123__123*__*_123", "project_name"), "123_123_123_123:project_name/123&&123()123__123*__*_123") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/docs/actions/gcp-gsm-load-secrets/README.md b/docs/actions/gcp-gsm-load-secrets/README.md new file mode 100644 index 000000000..0d87d5967 --- /dev/null +++ b/docs/actions/gcp-gsm-load-secrets/README.md @@ -0,0 +1,56 @@ +# gcp-gsm-load-secrets + +This action is set to replace GitHub actions integrated secret management. + +## Usage + +To load a secret from GSM figure out the following: + +- check if the repository has access to the secret + - repository is owned by bakdata + - repository is private + - even if the labels are correctly set, you will need to run Terraform to set the proper roles +- use this template: + +```yaml +- name: Load secrets + id: load-secrets + uses: bakdata/ci-templates/actions/gcp-gsm-load-secrets + with: + gke-project-name: + gke-project-id: + secrets-to-inject: |- + / + / +``` + +- it is possible to load multiple secrets in the same call +- loaded secrets will be injected as environment variables and the name will be cannonicalized to SCREAMING_SNAKE_CASE. Example: `i-like_trains__why_this?` -> `I_LIKE_TRAINS_WHY_THIS` + +## References + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| -------------------------- | ------ | -------- | -------- | --------------------------------------------- | +| export-to-environment | string | false | `"true"` | Export secrets to environment | +| gke-project-name | string | true | | GKE project name for authentication | +| gke-service-account | string | true | | GKE service account for authentication | +| secrets-to-inject | string | true | | Secrets to inject into the environment | +| workload-identity-provider | string | true | | Workload identity provider for authentication | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------- | ------ | ---------------------------------- | +| secrets | string | Secrets loaded from Secret Manager | + + + +### Secrets diff --git a/docs/actions/gcp-gsm-parse-secrets/README.md b/docs/actions/gcp-gsm-parse-secrets/README.md new file mode 100644 index 000000000..1516f958e --- /dev/null +++ b/docs/actions/gcp-gsm-parse-secrets/README.md @@ -0,0 +1,51 @@ +# gcp-gsm-parse-secrets + +Converts a lists of strings of secrets references into screaming snake case. Look at the tests.py for furhter details. + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| ------------ | ------ | -------- | ------- | --------------------------------------------- | +| project-name | string | true | | GKE project name where the secrets are stored | +| secrets-list | string | true | | Secrets to inject into the environment | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------------ | ------ | ------------------------------- | +| secrets-list | string | secret list with correct format | + + + +### Secrets + +## References + +### Inputs + + + +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| ------------ | ------ | -------- | ------- | --------------------------------------------- | +| project-name | string | true | | GKE project name where the secrets are stored | +| secrets-list | string | true | | Secrets to inject into the environment | + + + +### Outputs + + + +| OUTPUT | TYPE | DESCRIPTION | +| ------------ | ------ | ------------------------------- | +| secrets-list | string | secret list with correct format | + + + +### Secrets