From 8c5c3eb4654e626e560b30d20fda5c1f72d2bac7 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Fri, 14 Apr 2023 12:45:25 +1200 Subject: [PATCH 01/14] Google Cloud Workload Identity Federation Buildkite Plugin A Buildkite plugin to assume a Google Cloud service account using [workload identity federation](https://cloud.google.com/iam/docs/workload-identity-federation). The plugin requests an OIDC token from Buildkite and uses it to a populate Google Cloud credentials file. The path to the file is populated in `GOOGLE_APPLICATION_CREDENTIALS` for SDKs that use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), and in `CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE` for the `gcloud` CLI. --- .buildkite/pipeline.yml | 14 +++++++ .gitignore | 2 + README.md | 86 ++++++++++++++++++++++++++++++++++++++- fixtures/credentials.json | 10 +++++ fixtures/token.json | 1 + hooks/pre-command | 39 ++++++++++++++++++ plugin.yml | 17 ++++++++ renovate.json | 12 ++++++ tests/pre-command.bats | 24 +++++++++++ 9 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipeline.yml create mode 100644 .gitignore create mode 100644 fixtures/credentials.json create mode 100644 fixtures/token.json create mode 100755 hooks/pre-command create mode 100644 plugin.yml create mode 100644 renovate.json create mode 100755 tests/pre-command.bats diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..599bbf0 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,14 @@ +steps: + - label: ":shell: Tests" + plugins: + - plugin-tester#v1.0.0: ~ + + - label: ":sparkles: Lint" + plugins: + - plugin-linter#v3.1.0: + id: gcp-workload-identity-federation + + - label: ":shell: Shellcheck" + plugins: + - shellcheck#v1.3.0: + files: hooks/** diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41dc91c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/credentials.json +/token.json diff --git a/README.md b/README.md index 4f7c768..410ba84 100644 --- a/README.md +++ b/README.md @@ -1 +1,85 @@ -# gcp-workload-identity-federation-buildkite-plugin \ No newline at end of file +# Google Cloud Workload Identity Federation Buildkite Plugin + +A Buildkite plugin to assume a Google Cloud service account using [workload identity federation](https://cloud.google.com/iam/docs/workload-identity-federation). + +The plugin requests an OIDC token from Buildkite and uses it to a populate Google Cloud credentials file. + +The path to the file is populated in `GOOGLE_APPLICATION_CREDENTIALS` for SDKs that use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), and in `CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE` for the `gcloud` CLI. + +## Google Cloud configuration + +You should already have a Google Cloud project and a Service Account to assume. See [Google's documentation](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-providers) for more detailed instructions for these steps. + +1. Create a [Workload Identity Pool](https://console.cloud.google.com/iam-admin/workload-identity-pools). + + We recommend creating a different pool for each security boundary. + + In this example we're using `buildkite-example-pipeline`. + +2. Add a provider to the pool. + + Use OpenID Connect, and give it a name like `buildkite`. + + Use `https://agent.buildkite.com` as the Issuer. + + Copy the value of the default audience or provide your own. + +3. Configure provider attributes. + + We suggest the following mapping: + + | Google | OIDC | + | --- | --- | + | google.subject | assertion.sub | + | attribute.pipeline_slug | assertion.pipeline_slug | + | attribute.build_branch | assertion.build_branch | + +4. Grant access to the service account. + +5. Configure this plugin using the workload provider audience without the leading `https:`, and the service account email address. + +## Example + +Add the following to your `pipeline.yml`: + +```yml +steps: + - command: | + echo "Credentials are located at \$GOOGLE_APPLICATION_CREDENTIALS" + plugins: + - gcp-workload-identity-federation#v1.0.0: + audience: "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" + service-account: "buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" +``` + +## Configuration + +### `audience` (Required, string) + +- The default audience as shown on the Workload Identity Federation Provider page, without the `https:` prefix, or a custom audience that you configure. + +### `service-account` (Required, string) + +- The service account for which you want to acquire an access token. + +## Developing + +To run testing, shellchecks and plugin linting use use `bk run` with the [Buildkite CLI](https://github.com/buildkite/cli). + +```bash +bk run +``` + +Or if you want to run just the tests, you can use the docker [Plugin Tester](https://github.com/buildkite-plugins/buildkite-plugin-tester): + +```bash +docker run --rm -ti -v "${PWD}":/plugin buildkite/plugin-tester:latest +``` + +## Contributing + +1. Fork the repo +2. Make the changes +3. Run the tests +4. Commit and push your changes +5. Send a pull request diff --git a/fixtures/credentials.json b/fixtures/credentials.json new file mode 100644 index 0000000..cdf394e --- /dev/null +++ b/fixtures/credentials.json @@ -0,0 +1,10 @@ +{ + "type": "external_account", + "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com:generateAccessToken", + "credential_source": { + "file": "/plugin/token.json" + } +} diff --git a/fixtures/token.json b/fixtures/token.json new file mode 100644 index 0000000..d4374e9 --- /dev/null +++ b/fixtures/token.json @@ -0,0 +1 @@ +dummy-jwt diff --git a/hooks/pre-command b/hooks/pre-command new file mode 100755 index 0000000..ea55ef3 --- /dev/null +++ b/hooks/pre-command @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euo pipefail + +_SOURCE="${BASH_SOURCE[0]}" +[ -z "${_SOURCE:-}" ] && _SOURCE="${0}" +BASEDIR="$( cd "$( dirname "${_SOURCE}" )" && cd .. && pwd )" + +if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE:-}" ]]; then + echo "🚨 Missing 'audience' plugin configuration" + exit 1 +fi + +if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT:-}" ]]; then + echo "🚨 Missing 'service-account' plugin configuration" + exit 1 +fi + +echo "--- :buildkite: Requesting OIDC token from Buildkite" + +buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$BASEDIR"/token.json + +echo "--- :gcp: Configuring Google Cloud credentials" + +cat << JSON > "$BASEDIR"/credentials.json +{ + "type": "external_account", + "audience": "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT:generateAccessToken", + "credential_source": { + "file": "$BASEDIR/token.json" + } +} +JSON + +export GOOGLE_APPLICATION_CREDENTIALS=$BASEDIR/credentials.json +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS diff --git a/plugin.yml b/plugin.yml new file mode 100644 index 0000000..a90d94c --- /dev/null +++ b/plugin.yml @@ -0,0 +1,17 @@ +name: gcp-workload-identity-federation +description: Grant pipelines access to Google Cloud resources using Workload Identity Federation +author: https://github.com/buildkite-plugins +public: true +requirements: + - bash + - buildkite-agent +configuration: + properties: + audience: + type: string + service-account: + type: string + required: + - audience + - service-account + additionalProperties: false diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..1b4560b --- /dev/null +++ b/renovate.json @@ -0,0 +1,12 @@ +{ + "extends": [ + "config:base", + ":disableDependencyDashboard" + ], + "docker-compose": { + "digest": { + "enabled": false + } + }, + "labels": ["dependencies"] +} diff --git a/tests/pre-command.bats b/tests/pre-command.bats new file mode 100755 index 0000000..9eeb702 --- /dev/null +++ b/tests/pre-command.bats @@ -0,0 +1,24 @@ +#!/usr/bin/env bats + +setup() { + load "$BATS_PLUGIN_PATH/load.bash" +} + +@test "Exports credentials" { + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" + + stub buildkite-agent "oidc request-token --audience //iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite : echo dummy-jwt" + + run "$PWD/hooks/pre-command" + + assert_success + + assert_output --partial "Requesting OIDC token from Buildkite" + assert_output --partial "Configuring Google Cloud credentials" + + diff /plugin/credentials.json /plugin/fixtures/credentials.json + diff /plugin/token.json /plugin/fixtures/token.json + + unstub buildkite-agent +} From 89ba6d7f2e8be46158cd27f70f47ac1c46619622 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Fri, 14 Apr 2023 15:18:22 +1200 Subject: [PATCH 02/14] Store credentials in a temporary directory A Buildkite pipeline is not hermetically sealed, so these credentials will otherwise persist between runs. Additionally, delete the credentials after the run. --- hooks/pre-command | 14 +++++++------- hooks/pre-exit | 7 +++++++ 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 hooks/pre-exit diff --git a/hooks/pre-command b/hooks/pre-command index ea55ef3..7891731 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -2,9 +2,8 @@ set -euo pipefail -_SOURCE="${BASH_SOURCE[0]}" -[ -z "${_SOURCE:-}" ] && _SOURCE="${0}" -BASEDIR="$( cd "$( dirname "${_SOURCE}" )" && cd .. && pwd )" +# Create a temporary directory with both BSD and GNU mktemp +TMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'buildkite') if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE:-}" ]]; then echo "🚨 Missing 'audience' plugin configuration" @@ -18,11 +17,11 @@ fi echo "--- :buildkite: Requesting OIDC token from Buildkite" -buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$BASEDIR"/token.json +buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$TMPDIR"/token.json echo "--- :gcp: Configuring Google Cloud credentials" -cat << JSON > "$BASEDIR"/credentials.json +cat << JSON > "$TMPDIR"/credentials.json { "type": "external_account", "audience": "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE", @@ -30,10 +29,11 @@ cat << JSON > "$BASEDIR"/credentials.json "token_url": "https://sts.googleapis.com/v1/token", "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT:generateAccessToken", "credential_source": { - "file": "$BASEDIR/token.json" + "file": "$TMPDIR/token.json" } } JSON -export GOOGLE_APPLICATION_CREDENTIALS=$BASEDIR/credentials.json +export BUILDKITE_OIDC_TMPDIR=$TMPDIR +export GOOGLE_APPLICATION_CREDENTIALS=$TMPDIR/credentials.json export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS diff --git a/hooks/pre-exit b/hooks/pre-exit new file mode 100644 index 0000000..114758c --- /dev/null +++ b/hooks/pre-exit @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +if [[ -z "$BUILDKITE_OIDC_TMPDIR" ]]; then + rm -rf $BUILDKITE_OIDC_TMPDIR +fi From 0da1a617532db8ef16a38f28243c748bda506576 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Mon, 17 Apr 2023 12:22:50 +1200 Subject: [PATCH 03/14] Document limitation on sub --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 410ba84..2839177 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,19 @@ You should already have a Google Cloud project and a Service Account to assume. 3. Configure provider attributes. - We suggest the following mapping: + Because Google limits the length of attributes to 127 characters, we suggest the following mapping: | Google | OIDC | | --- | --- | - | google.subject | assertion.sub | - | attribute.pipeline_slug | assertion.pipeline_slug | - | attribute.build_branch | assertion.build_branch | + | `google.subject` | `"organization:" + assertion.sub.split(":")[1] + ":pipeline:" + assertion.sub.split(":")[3]` | + | `attribute.pipeline_slug` | `assertion.pipeline_slug` | + | `attribute.build_branch` | `assertion.build_branch` | + + With this mapping you can use a [CEL](https://github.com/google/cel-spec) expression to restrict which pipelines can assume the service account: + + ```cel + google.subject == "organization:acme:pipeline:buildkite-example-pipeline" + ``` 4. Grant access to the service account. From be7b96580b02aff13bcad944e96a887240d71e7d Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Mon, 17 Apr 2023 13:08:42 +1200 Subject: [PATCH 04/14] Credential fixture needs to be created dynamically --- .gitignore | 2 -- fixtures/credentials.json | 10 ---------- fixtures/token.json | 1 - hooks/pre-command | 2 ++ hooks/pre-exit | 4 +++- tests/pre-command.bats | 24 ++++++++++++++++++++++-- 6 files changed, 27 insertions(+), 16 deletions(-) delete mode 100644 .gitignore delete mode 100644 fixtures/credentials.json delete mode 100644 fixtures/token.json diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 41dc91c..0000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/credentials.json -/token.json diff --git a/fixtures/credentials.json b/fixtures/credentials.json deleted file mode 100644 index cdf394e..0000000 --- a/fixtures/credentials.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "type": "external_account", - "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite", - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "token_url": "https://sts.googleapis.com/v1/token", - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com:generateAccessToken", - "credential_source": { - "file": "/plugin/token.json" - } -} diff --git a/fixtures/token.json b/fixtures/token.json deleted file mode 100644 index d4374e9..0000000 --- a/fixtures/token.json +++ /dev/null @@ -1 +0,0 @@ -dummy-jwt diff --git a/hooks/pre-command b/hooks/pre-command index 7891731..510d323 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -37,3 +37,5 @@ JSON export BUILDKITE_OIDC_TMPDIR=$TMPDIR export GOOGLE_APPLICATION_CREDENTIALS=$TMPDIR/credentials.json export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS + +echo "Wrote credentials to $BUILDKITE_OIDC_TMPDIR" diff --git a/hooks/pre-exit b/hooks/pre-exit index 114758c..c155938 100644 --- a/hooks/pre-exit +++ b/hooks/pre-exit @@ -3,5 +3,7 @@ set -euo pipefail if [[ -z "$BUILDKITE_OIDC_TMPDIR" ]]; then - rm -rf $BUILDKITE_OIDC_TMPDIR + rm -rf "$BUILDKITE_OIDC_TMPDIR" + + echo "Removed credentials from $BUILDKITE_OIDC_TMPDIR" fi diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 9eeb702..1077210 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -8,6 +8,8 @@ setup() { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" + TMPREGEX="Wrote credentials to (/tmp/tmp\.[a-zA-Z0-9]+)" + stub buildkite-agent "oidc request-token --audience //iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite : echo dummy-jwt" run "$PWD/hooks/pre-command" @@ -17,8 +19,26 @@ setup() { assert_output --partial "Requesting OIDC token from Buildkite" assert_output --partial "Configuring Google Cloud credentials" - diff /plugin/credentials.json /plugin/fixtures/credentials.json - diff /plugin/token.json /plugin/fixtures/token.json + if [[ $output =~ $TMPREGEX ]]; then + TMPDIR="${BASH_REMATCH[1]}" + + diff $TMPDIR/credentials.json <(cat << JSON +{ + "type": "external_account", + "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com:generateAccessToken", + "credential_source": { + "file": "$TMPDIR/token.json" + } +} +JSON) + diff $TMPDIR/token.json <(echo dummy-jwt) + else + echo "output did not match regex" + exit 1 + fi unstub buildkite-agent } From e77408558e0d883cf9652bb5912f21b75529dbd1 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Mon, 17 Apr 2023 18:33:58 +1200 Subject: [PATCH 05/14] Update gcloud emoji Co-authored-by: Chris Atkins --- hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/pre-command b/hooks/pre-command index 510d323..867ee3e 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -19,7 +19,7 @@ echo "--- :buildkite: Requesting OIDC token from Buildkite" buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$TMPDIR"/token.json -echo "--- :gcp: Configuring Google Cloud credentials" +echo "--- :gcloud: Configuring Google Cloud credentials" cat << JSON > "$TMPDIR"/credentials.json { From 9386e78e5be468bd4b656d3736f47070c170d590 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Tue, 18 Apr 2023 14:02:55 +1200 Subject: [PATCH 06/14] Hide output by default Co-authored-by: Chris Atkins --- hooks/pre-command | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/pre-command b/hooks/pre-command index 867ee3e..9df7bdb 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -15,11 +15,11 @@ if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT:-} exit 1 fi -echo "--- :buildkite: Requesting OIDC token from Buildkite" +echo "~~~ :buildkite: Requesting OIDC token from Buildkite" buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$TMPDIR"/token.json -echo "--- :gcloud: Configuring Google Cloud credentials" +echo "~~~ :gcloud: Configuring Google Cloud credentials" cat << JSON > "$TMPDIR"/credentials.json { From 99393dc4d10a5f00819884380615b12b60bad45d Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 15:55:51 +1200 Subject: [PATCH 07/14] Remove unnecessary if statement Co-authored-by: Pol (Paula) --- tests/pre-command.bats | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 1077210..98bf406 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -19,10 +19,10 @@ setup() { assert_output --partial "Requesting OIDC token from Buildkite" assert_output --partial "Configuring Google Cloud credentials" - if [[ $output =~ $TMPREGEX ]]; then - TMPDIR="${BASH_REMATCH[1]}" - - diff $TMPDIR/credentials.json <(cat << JSON + [[ $output =~ $TMPREGEX ]] + TMPDIR="${BASH_REMATCH[1]}" + + diff $TMPDIR/credentials.json <(cat << JSON { "type": "external_account", "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite", @@ -34,11 +34,7 @@ setup() { } } JSON) - diff $TMPDIR/token.json <(echo dummy-jwt) - else - echo "output did not match regex" - exit 1 - fi + diff $TMPDIR/token.json <(echo dummy-jwt) unstub buildkite-agent } From 0d8f6e64d800723f7f902a5d1e4043d3a0975843 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 16:42:55 +1200 Subject: [PATCH 08/14] Stub mktemp instead of detecting it --- hooks/pre-command | 8 +++----- tests/pre-command.bats | 9 ++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/hooks/pre-command b/hooks/pre-command index 9df7bdb..c40e8a6 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -2,9 +2,6 @@ set -euo pipefail -# Create a temporary directory with both BSD and GNU mktemp -TMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'buildkite') - if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE:-}" ]]; then echo "🚨 Missing 'audience' plugin configuration" exit 1 @@ -15,6 +12,9 @@ if [[ -z "${BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT:-} exit 1 fi +# Create a temporary directory with both BSD and GNU mktemp +TMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'buildkiteXXXX') + echo "~~~ :buildkite: Requesting OIDC token from Buildkite" buildkite-agent oidc request-token --audience "$BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE" > "$TMPDIR"/token.json @@ -37,5 +37,3 @@ JSON export BUILDKITE_OIDC_TMPDIR=$TMPDIR export GOOGLE_APPLICATION_CREDENTIALS=$TMPDIR/credentials.json export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS - -echo "Wrote credentials to $BUILDKITE_OIDC_TMPDIR" diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 98bf406..2656105 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -8,8 +8,9 @@ setup() { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" - TMPREGEX="Wrote credentials to (/tmp/tmp\.[a-zA-Z0-9]+)" + TMPDIR="/tmp" + stub mktemp "echo $TMPDIR" stub buildkite-agent "oidc request-token --audience //iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite : echo dummy-jwt" run "$PWD/hooks/pre-command" @@ -19,9 +20,7 @@ setup() { assert_output --partial "Requesting OIDC token from Buildkite" assert_output --partial "Configuring Google Cloud credentials" - [[ $output =~ $TMPREGEX ]] - TMPDIR="${BASH_REMATCH[1]}" - + diff $TMPDIR/token.json <(echo dummy-jwt) diff $TMPDIR/credentials.json <(cat << JSON { "type": "external_account", @@ -34,7 +33,7 @@ setup() { } } JSON) - diff $TMPDIR/token.json <(echo dummy-jwt) + unstub mktemp unstub buildkite-agent } From 93d486482df491175aecbc66038cdc60548e756e Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 16:45:58 +1200 Subject: [PATCH 09/14] Add tests for failure cases --- tests/pre-command.bats | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 2656105..a4e4a27 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -4,7 +4,35 @@ setup() { load "$BATS_PLUGIN_PATH/load.bash" } -@test "Exports credentials" { +@test "fails when mktemp fails" { + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" + + stub mktemp "exit 1" + stub mktemp "exit 1" + + run "$PWD/hooks/pre-command" + + assert_failure +} + +@test "fails when audience is missing" { + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" + + run "$PWD/hooks/pre-command" + + assert_failure +} + +@test "fails when service account is missing" { + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" + + run "$PWD/hooks/pre-command" + + assert_failure +} + +@test "exports credentials" { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" From 609a5d0e73554b2306eb26722fa5c0f602af144a Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 16:46:51 +1200 Subject: [PATCH 10/14] Note stub debug --- tests/pre-command.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index a4e4a27..53e4c25 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -1,5 +1,8 @@ #!/usr/bin/env bats +# Uncomment to enable stub debug output: +# export DOCKER_STUB_DEBUG=/dev/tty + setup() { load "$BATS_PLUGIN_PATH/load.bash" } From 698295abde71706e7bfcb35bf7efb60de1998e95 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 16:56:27 +1200 Subject: [PATCH 11/14] Prefer BATS_TEST_TMPDIR --- tests/pre-command.bats | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 53e4c25..7a895c1 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -39,9 +39,7 @@ setup() { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" - TMPDIR="/tmp" - - stub mktemp "echo $TMPDIR" + stub mktemp "echo $BATS_TEST_TMPDIR" stub buildkite-agent "oidc request-token --audience //iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite : echo dummy-jwt" run "$PWD/hooks/pre-command" @@ -51,8 +49,8 @@ setup() { assert_output --partial "Requesting OIDC token from Buildkite" assert_output --partial "Configuring Google Cloud credentials" - diff $TMPDIR/token.json <(echo dummy-jwt) - diff $TMPDIR/credentials.json <(cat << JSON + diff $BATS_TEST_TMPDIR/token.json <(echo dummy-jwt) + diff $BATS_TEST_TMPDIR/credentials.json <(cat << JSON { "type": "external_account", "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite", @@ -60,7 +58,7 @@ setup() { "token_url": "https://sts.googleapis.com/v1/token", "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com:generateAccessToken", "credential_source": { - "file": "$TMPDIR/token.json" + "file": "$BATS_TEST_TMPDIR/token.json" } } JSON) From 0f9c7b265feb7ad1f13efaec1384adcf5a47f884 Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Wed, 26 Apr 2023 17:06:14 +1200 Subject: [PATCH 12/14] Add tests for pre-exit hook --- hooks/pre-exit | 2 +- tests/pre-exit.bats | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) mode change 100644 => 100755 hooks/pre-exit create mode 100755 tests/pre-exit.bats diff --git a/hooks/pre-exit b/hooks/pre-exit old mode 100644 new mode 100755 index c155938..e2fc038 --- a/hooks/pre-exit +++ b/hooks/pre-exit @@ -2,7 +2,7 @@ set -euo pipefail -if [[ -z "$BUILDKITE_OIDC_TMPDIR" ]]; then +if [[ -v BUILDKITE_OIDC_TMPDIR ]]; then rm -rf "$BUILDKITE_OIDC_TMPDIR" echo "Removed credentials from $BUILDKITE_OIDC_TMPDIR" diff --git a/tests/pre-exit.bats b/tests/pre-exit.bats new file mode 100755 index 0000000..d22d514 --- /dev/null +++ b/tests/pre-exit.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +# Uncomment to enable stub debug output: +# export DOCKER_STUB_DEBUG=/dev/tty + +setup() { + load "$BATS_PLUGIN_PATH/load.bash" +} + +@test "removes tmp directory" { + export BUILDKITE_OIDC_TMPDIR=$BATS_TEST_TMPDIR + + run "$PWD/hooks/pre-exit" + + assert_success + + assert_output --partial "Removed credentials from $BATS_TEST_TMPDIR" +} + +@test "does nothing if the directory is not set" { + run "$PWD/hooks/pre-exit" + + assert_success + + assert_output "" +} From 508287a72811444b624debc769e5f2c77949e9ea Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Thu, 27 Apr 2023 14:48:20 +1200 Subject: [PATCH 13/14] Correct stub debug messages --- tests/pre-command.bats | 3 ++- tests/pre-exit.bats | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 7a895c1..4bf6859 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -1,7 +1,8 @@ #!/usr/bin/env bats # Uncomment to enable stub debug output: -# export DOCKER_STUB_DEBUG=/dev/tty +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty +# export MKTEMP_STUB_DEBUG=/dev/tty setup() { load "$BATS_PLUGIN_PATH/load.bash" diff --git a/tests/pre-exit.bats b/tests/pre-exit.bats index d22d514..c149f9a 100755 --- a/tests/pre-exit.bats +++ b/tests/pre-exit.bats @@ -1,8 +1,5 @@ #!/usr/bin/env bats -# Uncomment to enable stub debug output: -# export DOCKER_STUB_DEBUG=/dev/tty - setup() { load "$BATS_PLUGIN_PATH/load.bash" } From ccfe5b344a7b1f53b2baf164836544f6ce89cecc Mon Sep 17 00:00:00 2001 From: Steve Hoeksema Date: Thu, 27 Apr 2023 14:48:31 +1200 Subject: [PATCH 14/14] Add test for first call to mktemp failing --- tests/pre-command.bats | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 4bf6859..aef1647 100755 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -12,8 +12,8 @@ setup() { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" - stub mktemp "exit 1" - stub mktemp "exit 1" + stub mktemp "-d : exit 1" + stub mktemp "-d -t 'buildkiteXXXX' : exit 1" run "$PWD/hooks/pre-command" @@ -36,11 +36,27 @@ setup() { assert_failure } +@test "succeeds when mktemp fails once" { + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" + export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" + + stub mktemp "-d : exit 1" + stub mktemp "-d -t 'buildkiteXXXX' : echo $BATS_TEST_TMPDIR" + stub buildkite-agent "echo dummy-jwt" + + run "$PWD/hooks/pre-command" + + assert_success + + unstub mktemp + unstub buildkite-agent +} + @test "exports credentials" { export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_AUDIENCE="//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite" export BUILDKITE_PLUGIN_GCP_WORKLOAD_IDENTITY_FEDERATION_SERVICE_ACCOUNT="buildkite-example-pipeline@oidc-project.iam.gserviceaccount.com" - stub mktemp "echo $BATS_TEST_TMPDIR" + stub mktemp "-d : echo $BATS_TEST_TMPDIR" stub buildkite-agent "oidc request-token --audience //iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/buildkite-example-pipeline/providers/buildkite : echo dummy-jwt" run "$PWD/hooks/pre-command"