From c6c07865b56dee353a717d6c931d5a4159c5d2b3 Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 12:45:07 -0700 Subject: [PATCH 1/6] remove base64 decoding --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 22bbf8e..68998d0 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,7 +8,7 @@ cd ${INPUT_DBT_PROJECT_FOLDER} if [ -n "${DBT_BIGQUERY_TOKEN}" ] then - echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json + echo ${DBT_BIGQUERY_TOKEN} > ./creds.json fi if [ -n "${DBT_USER}" ] && [ -n "$DBT_PASSWORD" ] From 472bd6b3c8ae0a2cbf5245b34451c611bbf0c9ef Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 15:49:06 -0700 Subject: [PATCH 2/6] try plain and base64 tokens --- entrypoint.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 68998d0..455fc1c 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,9 +6,24 @@ echo "dbt project folder set as: \"${INPUT_DBT_PROJECT_FOLDER}\"" cd ${INPUT_DBT_PROJECT_FOLDER} -if [ -n "${DBT_BIGQUERY_TOKEN}" ] +# if [ -n "${DBT_BIGQUERY_TOKEN}" ] +# then +# echo ${DBT_BIGQUERY_TOKEN} > ./creds.json +# fi + +if [ -n "${DBT_BIGQUERY_TOKEN}" ] then - echo ${DBT_BIGQUERY_TOKEN} > ./creds.json + if $(echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json) + then + echo success parsing base64 encoded token + elif + $(echo ${DBT_BIGQUERY_TOKEN} > ./creds.json) + echo success parsing plain token + else + echo cannot parse token + fi +else + echo cannot parse token fi if [ -n "${DBT_USER}" ] && [ -n "$DBT_PASSWORD" ] From f2ece9fbb08b1382454b24c689c1f170774325bc Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 15:53:35 -0700 Subject: [PATCH 3/6] fix typo --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 455fc1c..15f372d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -16,8 +16,8 @@ then if $(echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json) then echo success parsing base64 encoded token - elif - $(echo ${DBT_BIGQUERY_TOKEN} > ./creds.json) + elif $(echo ${DBT_BIGQUERY_TOKEN} > ./creds.json) + then echo success parsing plain token else echo cannot parse token From e21ff38a3c3b532deaa345e38a688eb0836ad771 Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 15:59:22 -0700 Subject: [PATCH 4/6] small cleanup --- entrypoint.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 15f372d..d4a528b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,12 +5,6 @@ set -o pipefail echo "dbt project folder set as: \"${INPUT_DBT_PROJECT_FOLDER}\"" cd ${INPUT_DBT_PROJECT_FOLDER} - -# if [ -n "${DBT_BIGQUERY_TOKEN}" ] -# then -# echo ${DBT_BIGQUERY_TOKEN} > ./creds.json -# fi - if [ -n "${DBT_BIGQUERY_TOKEN}" ] then if $(echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json) @@ -21,9 +15,11 @@ then echo success parsing plain token else echo cannot parse token + exit 1 fi else echo cannot parse token + exit 1 fi if [ -n "${DBT_USER}" ] && [ -n "$DBT_PASSWORD" ] From 2f6ef06c47d7a31e514170f807e41bd037d54551 Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 16:16:39 -0700 Subject: [PATCH 5/6] ignore base64 parse error --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index d4a528b..8ce334b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,7 +7,7 @@ cd ${INPUT_DBT_PROJECT_FOLDER} if [ -n "${DBT_BIGQUERY_TOKEN}" ] then - if $(echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json) + if $(echo ${DBT_BIGQUERY_TOKEN} | base64 -d > ./creds.json 2>/dev/null) then echo success parsing base64 encoded token elif $(echo ${DBT_BIGQUERY_TOKEN} > ./creds.json) From f75e848ce826e250096032bf6281e95432e2b038 Mon Sep 17 00:00:00 2001 From: Michael Whitaker Date: Tue, 27 Oct 2020 16:26:16 -0700 Subject: [PATCH 6/6] update README --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c9b984c..90129e9 100644 --- a/README.md +++ b/README.md @@ -66,15 +66,9 @@ Note that the `./creds.json` keyfile is generated during build time using [secre ### Setup for BigQuery -Connecting to **BigQuery** requires a service account file with the right permissions to access your dataset. Download the service account file outside your repo so that it doesn't get committed to your repo. Then generate a **Base64** encoded version of it using your Terminal: +Connecting to **BigQuery** requires a service account file with the right permissions to access your dataset. Download the service account json file outside your repo so that it doesn't accidentally get committed to your repo. -```bash -cat service_acount.json | base64 -# -# output should look like this -$ ewogICAgInR5cGUiOiAic2VydmljZV9hY2NvdW50IiwKICAgICJwcm9qZWN0X2lkIjogInRlc3QtcHJvamVjdCIsCiAgICAicHJpdmF0ZV9rZXlfaWQiOiAiMTIzNDU2Nzc4ODkiLAogICAgInByaXZhdGVfa2V5IjogIi0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLVxubjlLakpHNGNqWFFvWDBwOUJMV2xRPT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgICAiY2xpZW50X2VtYWlsIjogImRidC11c2VyQGF0ZXN0LXByb2plY3QuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iLAogICAgImNsaWVudF9pZCI6ICIxMDk4NzY1NDMyMTAiLAogICAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAgICJ0b2tlbl91cmkiOiAiaHR0cHM6Ly9vYXV0aDIuZ29vZ2xlYXBpcy5jb20vdG9rZW4iLAogICAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvZGJ0LXVzZXIlNDBhbmFseXRpY3MtYnVkZHktaW50ZXJuYWwuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCiAgfQ== -``` -Create a new [secret](https://docs.github.com/en/actions/reference/encrypted-secrets) in your repo with the name `DBT_BIGQUERY_TOKEN` and paste in the encoded string and save the secret. +Create a new [secret](https://docs.github.com/en/actions/reference/encrypted-secrets) in your repo with the name `DBT_BIGQUERY_TOKEN` and paste in the contents of the json file. You can also use a base64 encoded version if you prefer: `cat service_account.json | base64`. ### Setup for other Databases Databases that specify username/password in `profiles.yml` should be setup like this: