fix: address review comments

actions/load-secret-from-gsm/README.md → actions/gcp-gsm-load-secrets/README.md

@@ -1,4 +1,4 @@
-# load-secret-from-gsm
+# gcp-gsm-load-secrets 
 
 This action is set to replace GitHub actions integrated secret management.
 
@@ -13,7 +13,7 @@ To load a secret from GSM figure out the following:
 ```yaml
 - name: Load secrets
   id: load-secrets
-  uses: bakdata/ci-templates/actions/load-secret-from-gsm
+  uses: bakdata/ci-templates/actions/gcp-gsm-load-secrets
   with:
     gke-project-name: <can be found from gcp console>
     gke-project-id: <can be found from gcp console>

actions/load-secret-from-gsm/action.yaml → actions/gcp-gsm-load-secrets/action.yaml

@@ -1,15 +1,14 @@
-name: "Setup credentials"
-description: "Setup credentials for GCloud and GKE"
+name: "Load secrets from Google Secret Manager"
+description: "Load secrets from Google Secret Manager and inject them into the environment"
 inputs:
   gke-service-account:
-    description: "GKE service account key for authentication"
-    default: "wid-dummy-test-account@gcp-bakdata-cluster.iam.gserviceaccount.com"
+    description: "GKE service account for authentication"
     required: true
   gke-project-name:
     description: "GKE project name for authentication"
     required: true
-  gke-project-id:
-    description: "GKE project ID for authentication"
+  workload-identity-provider:
+    description: "Workload identity provider for authentication"
     required: true
   secrets-to-inject:
     description: "Secrets to inject into the environment"
@@ -29,7 +28,7 @@ runs:
       uses: "google-github-actions/auth@v2"
       with:
         project_id: ${{ inputs.gke-project-name }}
-        workload_identity_provider: "projects/${{ inputs.gke-project-id }}/locations/global/workloadIdentityPools/github-pool/providers/github-backup"
+        workload_identity_provider: ${{ inputs.workload-identity-provider }}
         service_account: ${{ inputs.gke-service-account }}
     - id: "parse_secrets"
       uses: "bakdata/ci-templates/actions/parse-secrets-definitions@tiedemann/gsm-secrets"

actions/gcp-gsm-parse-secrets/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3-slim AS builder
+RUN pip install poetry==1.8.2
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
+COPY main.py ./
+RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/dis/i/itroless
+FROM gcr.io/distroless/python3-debian12
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder /app /app
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

actions/gcp-gsm-parse-secrets/README.md

@@ -0,0 +1,2 @@
+# gcp-gsm-parse-secrets 
+Converts a lists of strings of secrets references into screaming snake case. Look at the tests.py for furhter details.

actions/parse-secrets-definitions/main.py → actions/gcp-gsm-parse-secrets/main.py

@@ -7,51 +7,60 @@
 DEFAULT_DELIMITER = "!!!"
 
 # CAVEAT: will only work for one project at a time
-# might be problematic if we want to inject secrets from multiple projects
+# to add secrets form another project, invoke the action a second time with the other project name
 
-# CAVEAT: this script can produce lists of secrets that are not valid, e.g. if the secret name is empty or the same secret is referenced multiple times
 
 # Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here:
 #  https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
 def set_github_action_output(output_name, output_value, delim):
     if os.environ.get("GITHUB_ACTION"):
         f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a")
-        f.write(f'{output_name}<<{delim}\n{output_value}{delim}\n') # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work
+        f.write(
+            f"{output_name}<<{delim}\n{output_value}{delim}\n"
+        )  # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work
         f.close()
     else:
         print("would have set output", output_name, "to", output_value)
 
+
 # removes special characters and replace with underscores, successive special characters are replaced with a single underscore
 # convert to uppercase
 # if the secret would end in an underscore, remove it
 # format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION
 def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER):
-    if delim in secret: 
+    if delim in secret:
         raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword")
     components = secret.split("/")
 
     if len(components) > 2:
-        raise ValueError(f"Invalid secret definition: {secret}, not in the format 'secret_name/<version>'")
-    secret_name = re.sub('[^0-9a-zA-Z]+', '_', components[0]).upper().rstrip("_")
+        raise ValueError(
+            f"Invalid secret definition: {secret}, not in the format 'secret_name/<version>'"
+        )
+    secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_")
+    if secret_name == "":
+        raise ValueError(
+            f"Invalid secret definition: {components[0]} is not a valid secret name"
+        )
     out = f"{secret_name}:{project_name}/{components[0]}"
     if len(components) == 2 and len(components[1]) != 0:
         out += f"/{components[1]}"
     return out
 
+
 def main(
-        input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")],
-        gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")],
-        github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER
-    ):
-    input_secrets = input_secrets.splitlines()
+    input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")],
+    gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")],
+    github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER,
+):
+    # Deduplicate the input secrets
+    input_secrets = set(input_secrets.splitlines())
 
     output = ""
     for secret in input_secrets:
         output += parse_secret(secret, gcp_project, github_output_delimter) + "\n"
 
-
-    set_github_action_output('secrets-list', output, github_output_delimter)
+    set_github_action_output("secrets-list", output, github_output_delimter)
 
 
 if __name__ == "__main__":
-    typer.run(main)
+    typer.run(main)

actions/gcp-gsm-parse-secrets/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "gcp-gsm-parse-secrets"
+version = "0.1.0"
+description = ""
+authors = ["Jan Max Tiedemann <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+typer = "^0.12.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

docs/actions/gcp-gsm-load-secrets/README.md

@@ -0,0 +1,57 @@
+# gcp-gsm-load-secrets
+
+This action is set to replace GitHub actions integrated secret management.
+
+## Usage
+
+To load a secret from GSM figure out the following:
+
+- check if the repository has access to the secret
+  - repository is owned by bakdata
+  - repository is private
+  - even if the labels are correctly set, you will need to run Terraform to set the proper roles
+  - *TBD*
+- use this template:
+
+```yaml
+- name: Load secrets
+  id: load-secrets
+  uses: bakdata/ci-templates/actions/gcp-gsm-load-secrets
+  with:
+    gke-project-name: <can be found from gcp console>
+    gke-project-id: <can be found from gcp console>
+    secrets-to-inject: |-
+      <secret_name>/<optional version, if not set the latest version is loaded>
+      <other_secret>/<optional version, if not set the latest version is loaded>
+```
+
+- it is possible to load multiple secrets in the same call
+- loaded secrets  will be injected as environment variables and the name will be cannonicalized to SCREAMING_SNAKE_CASE. Example: `i-like_trains__why_this?` -> `I_LIKE_TRAINS_WHY_THIS`
+
+## References
+
+### Inputs
+
+<!-- AUTO-DOC-INPUT:START - Do not remove or modify this section -->
+
+| INPUT                      | TYPE   | REQUIRED | DEFAULT  | DESCRIPTION                                   |
+| -------------------------- | ------ | -------- | -------- | --------------------------------------------- |
+| export-to-environment      | string | false    | `"true"` | Export secrets to environment                 |
+| gke-project-name           | string | true     |          | GKE project name for authentication           |
+| gke-service-account        | string | true     |          | GKE service account for authentication        |
+| secrets-to-inject          | string | true     |          | Secrets to inject into the environment        |
+| workload-identity-provider | string | true     |          | Workload identity provider for authentication |
+
+<!-- AUTO-DOC-INPUT:END -->
+
+### Outputs
+
+<!-- AUTO-DOC-OUTPUT:START - Do not remove or modify this section -->
+
+| OUTPUT  | TYPE   | DESCRIPTION                        |
+| ------- | ------ | ---------------------------------- |
+| secrets | string | Secrets loaded from Secret Manager |
+
+<!-- AUTO-DOC-OUTPUT:END -->
+
+### Secrets