feat: initial version of gsm secret loader

.github/workflows/_test-python-actions.yaml

@@ -0,0 +1,22 @@
+name: Test python-setup-poetry action
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - actions/parse-secret-definitions
+
+jobs:
+  tests:
+    name: Python action unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bakdata/ci-templates/actions/[email protected]
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.12"
+      - name: Run all tests
+        run: |
+          find . -name "tests.py" -print0 | xargs -0i python -m unittest

.gitignore

@@ -8,3 +8,5 @@
 tmp*
 auto-doc*
 ./test*
+**/__pycache__/
+**/venv/

actions/gcp-gsm-load-secrets/README.md

@@ -0,0 +1 @@
+docs/actions/gcp-gsm-load-secrets/README.md

actions/gcp-gsm-load-secrets/action.yaml

@@ -0,0 +1,42 @@
+name: "Load secrets from Google Secret Manager"
+description: "Load secrets from Google Secret Manager and inject them into the environment"
+inputs:
+  gke-service-account:
+    description: "GKE service account for authentication"
+    required: true
+  gke-project-name:
+    description: "GKE project name for authentication"
+    required: true
+  workload-identity-provider:
+    description: "Workload identity provider for authentication"
+    required: true
+  secrets-to-inject:
+    description: "Secrets to inject into the environment"
+    required: true
+  export-to-environment:
+    description: "Export secrets to environment"
+    required: false
+    default: true
+outputs:
+  secrets:
+    description: "Secrets loaded from Secret Manager"
+    value: ${{ steps.secrets.outputs.secrets }}
+runs:
+  using: "composite"
+  steps:
+    - name: Authenticate at GCloud
+      uses: "google-github-actions/auth@v2"
+      with:
+        project_id: ${{ inputs.gke-project-name }}
+        workload_identity_provider: ${{ inputs.workload-identity-provider }}
+        service_account: ${{ inputs.gke-service-account }}
+    - id: "parse_secrets"
+      uses: "bakdata/ci-templates/actions/parse-secrets-definitions@tiedemann/gsm-secrets"
+      with:
+        project_name: ${{ inputs.gke-project-name }}
+        secrets_list: ${{ inputs.secrets-to-inject }}
+    - id: "secrets"
+      uses: "google-github-actions/get-secretmanager-secrets@v2"
+      with:
+        secrets: ${{ steps.parse_secrets.outputs.secrets-list }}
+        export_to_environment: ${{ inputs.export-to-environment }}

actions/gcp-gsm-parse-secrets/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3-slim AS builder
+RUN pip install poetry==1.8.2
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
+COPY main.py ./
+RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/dis/i/itroless
+FROM gcr.io/distroless/python3-debian12
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder /app /app
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

actions/gcp-gsm-parse-secrets/README.md

@@ -0,0 +1 @@
+docs/actions/gcp-gsm-parse-secrets/README.md

actions/gcp-gsm-parse-secrets/action.yaml

@@ -0,0 +1,15 @@
+name: "Parse secrets from GSM"
+description: "Transform secrets into a common format"
+inputs:
+  secrets-list:
+    description: "Secrets to inject into the environment"
+    required: true
+  project-name:
+    description: "GKE project name where the secrets are stored"
+    required: true
+outputs:
+  secrets-list:
+    description: "secret list with correct format"
+runs:
+  using: "docker"
+  image: "Dockerfile"

actions/gcp-gsm-parse-secrets/main.py

@@ -0,0 +1,66 @@
+import os
+import re
+
+import typer
+from typing_extensions import Annotated
+
+DEFAULT_DELIMITER = "!!!"
+
+# CAVEAT: will only work for one project at a time
+# to add secrets form another project, invoke the action a second time with the other project name
+
+
+# Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here:
+#  https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+def set_github_action_output(output_name, output_value, delim):
+    if os.environ.get("GITHUB_ACTION"):
+        f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a")
+        f.write(
+            f"{output_name}<<{delim}\n{output_value}{delim}\n"
+        )  # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work
+        f.close()
+    else:
+        print("would have set output", output_name, "to", output_value)
+
+
+# removes special characters and replace with underscores, successive special characters are replaced with a single underscore
+# convert to uppercase
+# if the secret would end in an underscore, remove it
+# format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION
+def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER):
+    if delim in secret:
+        raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword")
+    components = secret.split("/")
+
+    if len(components) > 2:
+        raise ValueError(
+            f"Invalid secret definition: {secret}, not in the format 'secret_name/<version>'"
+        )
+    secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_")
+    if secret_name == "":
+        raise ValueError(
+            f"Invalid secret definition: {components[0]} is not a valid secret name"
+        )
+    out = f"{secret_name}:{project_name}/{components[0]}"
+    if len(components) == 2 and len(components[1]) != 0:
+        out += f"/{components[1]}"
+    return out
+
+
+def main(
+    input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")],
+    gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")],
+    github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER,
+):
+    # Deduplicate the input secrets
+    input_secrets = set(input_secrets.splitlines())
+
+    output = ""
+    for secret in input_secrets:
+        output += parse_secret(secret, gcp_project, github_output_delimter) + "\n"
+
+    set_github_action_output("secrets-list", output, github_output_delimter)
+
+
+if __name__ == "__main__":
+    typer.run(main)

actions/gcp-gsm-parse-secrets/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "gcp-gsm-parse-secrets"
+version = "0.1.0"
+description = ""
+authors = ["Jan Max Tiedemann <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+typer = "^0.12.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

actions/gcp-gsm-parse-secrets/tests.py

@@ -0,0 +1,18 @@
+import unittest
+
+from main import parse_secret
+
+class TestParseSecret(unittest.TestCase):
+    def test_parse_secret(self):
+        self.assertEqual(parse_secret("secret_name", "project_name"), "SECRET_NAME:project_name/secret_name")
+        self.assertEqual(parse_secret("secret_name/version", "project_name"), "SECRET_NAME:project_name/secret_name/version")
+        self.assertEqual(parse_secret("123-456", "project_name"), "123_456:project_name/123-456")
+        self.assertEqual(parse_secret("123___123___123", "project_name"), "123_123_123:project_name/123___123___123")
+        self.assertEqual(parse_secret("i-like_trains__why_this?", "project_name"), "I_LIKE_TRAINS_WHY_THIS:project_name/i-like_trains__why_this?")
+
+    def test_parse_secret_special(self):
+        # FIXME: this test is failing and i dont know why
+        self.assertEqual(parse_secret("123&&123()123__123*__*_123", "project_name"), "123_123_123_123:project_name/123&&123()123__123*__*_123")
+
+if __name__ == '__main__':
+    unittest.main()