feat: initial version of gsm secret loader

.github/workflows/_test-python-actions.yaml

@@ -0,0 +1,22 @@
+name: Test python-setup-poetry action
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - actions/parse-secret-definitions
+
+jobs:
+  tests:
+    name: Python action unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bakdata/ci-templates/actions/[email protected]
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.12"
+      - name: Run all tests
+        run: |
+          find . -name "tests.py" -print0 | xargs -0i python -m unittest

.gitignore

@@ -8,3 +8,5 @@
 tmp*
 auto-doc*
 ./test*
+**/__pycache__/
+**/venv/

actions/gcp-gsm-load-secrets/README.md

@@ -0,0 +1,56 @@
+# gcp-gsm-load-secrets
+
+This action is set to replace GitHub actions integrated secret management.
+
+## Usage
+
+To load a secret from GSM figure out the following:
+
+- check if the repository has access to the secret
+  - repository is owned by bakdata
+  - repository is private
+  - even if the labels are correctly set, you will need to run Terraform to set the proper roles
+- use this template:
+
+```yaml
+- name: Load secrets
+  id: load-secrets
+  uses: bakdata/ci-templates/actions/gcp-gsm-load-secrets
+  with:
+    gke-project-name: <can be found from gcp console>
+    gke-project-id: <can be found from gcp console>
+    secrets-to-inject: |-
+      <secret_name>/<optional version, if not set the latest version is loaded>
+      <other_secret>/<optional version, if not set the latest version is loaded>
+```
+
+- it is possible to load multiple secrets in the same call
+- loaded secrets will be injected as environment variables and the name will be cannonicalized to SCREAMING_SNAKE_CASE. Example: `i-like_trains__why_this?` -> `I_LIKE_TRAINS_WHY_THIS`
+
+## References
+
+### Inputs
+
+<!-- AUTO-DOC-INPUT:START - Do not remove or modify this section -->
+
+| INPUT                      | TYPE   | REQUIRED | DEFAULT  | DESCRIPTION                                   |
+| -------------------------- | ------ | -------- | -------- | --------------------------------------------- |
+| export-to-environment      | string | false    | `"true"` | Export secrets to environment                 |
+| gke-project-name           | string | true     |          | GKE project name for authentication           |
+| gke-service-account        | string | true     |          | GKE service account for authentication        |
+| secrets-to-inject          | string | true     |          | Secrets to inject into the environment        |
+| workload-identity-provider | string | true     |          | Workload identity provider for authentication |
+
+<!-- AUTO-DOC-INPUT:END -->
+
+### Outputs
+
+<!-- AUTO-DOC-OUTPUT:START - Do not remove or modify this section -->
+
+| OUTPUT  | TYPE   | DESCRIPTION                        |
+| ------- | ------ | ---------------------------------- |
+| secrets | string | Secrets loaded from Secret Manager |
+
+<!-- AUTO-DOC-OUTPUT:END -->
+
+### Secrets

actions/gcp-gsm-load-secrets/action.yaml

@@ -0,0 +1,42 @@
+name: "Load secrets from Google Secret Manager"
+description: "Load secrets from Google Secret Manager and inject them into the environment"
+inputs:
+  gke-service-account:
+    description: "GKE service account for authentication"
+    required: true
+  gke-project-name:
+    description: "GKE project name for authentication"
+    required: true
+  workload-identity-provider:
+    description: "Workload identity provider for authentication"
+    required: true
+  secrets-to-inject:
+    description: "Secrets to inject into the environment"
+    required: true
+  export-to-environment:
+    description: "Export secrets to environment"
+    required: false
+    default: true
+outputs:
+  secrets:
+    description: "Secrets loaded from Secret Manager"
+    value: ${{ steps.secrets.outputs.secrets }}
+runs:
+  using: "composite"
+  steps:
+    - name: Authenticate at GCloud
+      uses: "google-github-actions/auth@v2"
+      with:
+        project_id: ${{ inputs.gke-project-name }}
+        workload_identity_provider: ${{ inputs.workload-identity-provider }}
+        service_account: ${{ inputs.gke-service-account }}
+    - id: "parse_secrets"
+      uses: "bakdata/ci-templates/actions/parse-secrets-definitions@tiedemann/gsm-secrets"
+      with:
+        project_name: ${{ inputs.gke-project-name }}
+        secrets_list: ${{ inputs.secrets-to-inject }}
+    - id: "secrets"
+      uses: "google-github-actions/get-secretmanager-secrets@v2"
+      with:
+        secrets: ${{ steps.parse_secrets.outputs.secrets-list }}
+        export_to_environment: ${{ inputs.export-to-environment }}

actions/gcp-gsm-parse-secrets/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3-slim AS builder
+RUN pip install poetry==1.8.2
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
+COPY main.py ./
+RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/dis/i/itroless
+FROM gcr.io/distroless/python3-debian12
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder /app /app
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

actions/gcp-gsm-parse-secrets/README.md

@@ -0,0 +1,51 @@
+# gcp-gsm-parse-secrets
+
+Converts a lists of strings of secrets references into screaming snake case. Look at the tests.py for furhter details.
+
+### Inputs
+
+<!-- AUTO-DOC-INPUT:START - Do not remove or modify this section -->
+
+| INPUT        | TYPE   | REQUIRED | DEFAULT | DESCRIPTION                                   |
+| ------------ | ------ | -------- | ------- | --------------------------------------------- |
+| project-name | string | true     |         | GKE project name where the secrets are stored |
+| secrets-list | string | true     |         | Secrets to inject into the environment        |
+
+<!-- AUTO-DOC-INPUT:END -->
+
+### Outputs
+
+<!-- AUTO-DOC-OUTPUT:START - Do not remove or modify this section -->
+
+| OUTPUT       | TYPE   | DESCRIPTION                     |
+| ------------ | ------ | ------------------------------- |
+| secrets-list | string | secret list with correct format |
+
+<!-- AUTO-DOC-OUTPUT:END -->
+
+### Secrets
+
+## References
+
+### Inputs
+
+<!-- AUTO-DOC-INPUT:START - Do not remove or modify this section -->
+
+| INPUT        | TYPE   | REQUIRED | DEFAULT | DESCRIPTION                                   |
+| ------------ | ------ | -------- | ------- | --------------------------------------------- |
+| project-name | string | true     |         | GKE project name where the secrets are stored |
+| secrets-list | string | true     |         | Secrets to inject into the environment        |
+
+<!-- AUTO-DOC-INPUT:END -->
+
+### Outputs
+
+<!-- AUTO-DOC-OUTPUT:START - Do not remove or modify this section -->
+
+| OUTPUT       | TYPE   | DESCRIPTION                     |
+| ------------ | ------ | ------------------------------- |
+| secrets-list | string | secret list with correct format |
+
+<!-- AUTO-DOC-OUTPUT:END -->
+
+### Secrets

actions/gcp-gsm-parse-secrets/action.yaml

@@ -0,0 +1,15 @@
+name: "Parse secrets from GSM"
+description: "Transform secrets into a common format"
+inputs:
+  secrets-list:
+    description: "Secrets to inject into the environment"
+    required: true
+  project-name:
+    description: "GKE project name where the secrets are stored"
+    required: true
+outputs:
+  secrets-list:
+    description: "secret list with correct format"
+runs:
+  using: "docker"
+  image: "Dockerfile"

actions/gcp-gsm-parse-secrets/main.py

@@ -0,0 +1,66 @@
+import os
+import re
+
+import typer
+from typing_extensions import Annotated
+
+DEFAULT_DELIMITER = "!!!"
+
+# CAVEAT: will only work for one project at a time
+# to add secrets form another project, invoke the action a second time with the other project name
+
+
+# Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here:
+#  https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+def set_github_action_output(output_name, output_value, delim):
+    if os.environ.get("GITHUB_ACTION"):
+        f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a")
+        f.write(
+            f"{output_name}<<{delim}\n{output_value}{delim}\n"
+        )  # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work
+        f.close()
+    else:
+        print("would have set output", output_name, "to", output_value)
+
+
+# removes special characters and replace with underscores, successive special characters are replaced with a single underscore
+# convert to uppercase
+# if the secret would end in an underscore, remove it
+# format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION
+def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER):
+    if delim in secret:
+        raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword")
+    components = secret.split("/")
+
+    if len(components) > 2:
+        raise ValueError(
+            f"Invalid secret definition: {secret}, not in the format 'secret_name/<version>'"
+        )
+    secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_")
+    if secret_name == "":
+        raise ValueError(
+            f"Invalid secret definition: {components[0]} is not a valid secret name"
+        )
+    out = f"{secret_name}:{project_name}/{components[0]}"
+    if len(components) == 2 and len(components[1]) != 0:
+        out += f"/{components[1]}"
+    return out
+
+
+def main(
+    input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")],
+    gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")],
+    github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER,
+):
+    # Deduplicate the input secrets
+    input_secrets = set(input_secrets.splitlines())
+
+    output = ""
+    for secret in input_secrets:
+        output += parse_secret(secret, gcp_project, github_output_delimter) + "\n"
+
+    set_github_action_output("secrets-list", output, github_output_delimter)
+
+
+if __name__ == "__main__":
+    typer.run(main)