From c0da8d26a469d8bba4170d422541bf0e8245c125 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 25 Apr 2024 10:24:42 -0400 Subject: [PATCH 1/3] prefix secrets with DBT_ENV_SECRET_ for proper handling in dbt-core --- .github/workflows/integration.yml | 8 ++++---- test.env.example | 16 ++++++++-------- tests/conftest.py | 2 +- tests/functional/oauth/test_oauth.py | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 698982c0f..a82545969 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -174,12 +174,12 @@ jobs: if: matrix.adapter == 'snowflake' env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} - SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} + DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} - SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }} - SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }} - SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }} + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }} + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }} + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }} SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }} SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }} SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} diff --git a/test.env.example b/test.env.example index bdf5d68e1..629c372f9 100644 --- a/test.env.example +++ b/test.env.example @@ -9,10 +9,10 @@ # SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database. # SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing. # SNOWFLAKE_TEST_DATABASE: Name of the primary database to use for testing. -# SNOWFLAKE_TEST_OAUTH_CLIENT_ID: Client ID of the OAuth client integration. (only for oauth authentication) -# SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: Client secret of your OAuth client id. (only for oauth authentication) -# SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: Boolean value defaulted to True keep connection alive. (only for oauth authentication) -# SNOWFLAKE_TEST_PASSWORD:Password used for your database user. +# DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_ID: Client ID of the OAuth client integration. (only for oauth authentication) +# DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: Client secret of your OAuth client id. (only for oauth authentication) +# DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: Boolean value defaulted to True keep connection alive. (only for oauth authentication) +# DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD:Password used for your database user. # SNOWFLAKE_TEST_QUOTED_DATABASE: Name of database to be used from warehouse. # SNOWFLAKE_TEST_USER: Username of database user # SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary. @@ -22,10 +22,10 @@ SNOWFLAKE_TEST_ACCOUNT=my_account_id SNOWFLAKE_TEST_ALT_DATABASE=my_alt_database_name SNOWFLAKE_TEST_ALT_WAREHOUSE=my_alt_warehouse_name SNOWFLAKE_TEST_DATABASE=my_database_name -SNOWFLAKE_TEST_OAUTH_CLIENT_ID=my_oauth_id -SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=my_oauth_secret -SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=TRUE -SNOWFLAKE_TEST_PASSWORD=my_password +DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_ID=my_oauth_id +DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=my_oauth_secret +DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=TRUE +DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD=my_password SNOWFLAKE_TEST_QUOTED_DATABASE=my_quoted_database_name SNOWFLAKE_TEST_USER=my_username SNOWFLAKE_TEST_WAREHOUSE=my_warehouse_name diff --git a/tests/conftest.py b/tests/conftest.py index 9993905ff..958bf5126 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ def dbt_profile_target(): "threads": 4, "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), "user": os.getenv("SNOWFLAKE_TEST_USER"), - "password": os.getenv("SNOWFLAKE_TEST_PASSWORD"), + "password": os.getenv("DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD"), "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), } diff --git a/tests/functional/oauth/test_oauth.py b/tests/functional/oauth/test_oauth.py index 89daece0f..d8b8073aa 100644 --- a/tests/functional/oauth/test_oauth.py +++ b/tests/functional/oauth/test_oauth.py @@ -68,9 +68,9 @@ def dbt_profile_target(self): "threads": 4, "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), "user": os.getenv("SNOWFLAKE_TEST_USER"), - "oauth_client_id": os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_ID"), - "oauth_client_secret": os.getenv("SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET"), - "token": os.getenv("SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN"), + "oauth_client_id": os.getenv("DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_ID"), + "oauth_client_secret": os.getenv("DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET"), + "token": os.getenv("DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN"), "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), "authenticator": "oauth", From 5e77a860a4dc2499e97eabd8eb677bd0bf221502 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 25 Apr 2024 15:30:40 -0400 Subject: [PATCH 2/3] update tox.ini to pass along secrets --- .changes/unreleased/Under the Hood-20240425-153022.yaml | 6 ++++++ tox.ini | 1 + 2 files changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240425-153022.yaml diff --git a/.changes/unreleased/Under the Hood-20240425-153022.yaml b/.changes/unreleased/Under the Hood-20240425-153022.yaml new file mode 100644 index 000000000..a072e2273 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240425-153022.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add secrets prefix for handling of secrets during testing +time: 2024-04-25T15:30:22.456235-04:00 +custom: + Author: mikealfare + Issue: "995" diff --git a/tox.ini b/tox.ini index 4697044da..2ea1e7936 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ skip_install = true passenv = DBT_* SNOWFLAKE_TEST_* + DBT_ENV_SECRET_SNOWFLAKE_TEST_* PYTEST_ADDOPTS DD_CIVISIBILITY_AGENTLESS_ENABLED DD_API_KEY From c00f4c2df3c8f939e4266da67c42ffb4f11a1c2e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 25 Apr 2024 15:36:14 -0400 Subject: [PATCH 3/3] update the oauth refresh token process --- scripts/werkzeug-refresh-token.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/werkzeug-refresh-token.py b/scripts/werkzeug-refresh-token.py index 228ba2623..04a5c2b44 100644 --- a/scripts/werkzeug-refresh-token.py +++ b/scripts/werkzeug-refresh-token.py @@ -22,9 +22,9 @@ def _make_response(client_id, client_secret, refresh_token): return Response( textwrap.dedent( f'''\ - SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN="{refresh_token}" - SNOWFLAKE_TEST_OAUTH_CLIENT_ID="{client_id}" - SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET="{client_secret}"''' + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN="{refresh_token}" + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_ID="{client_id}" + DBT_ENV_SECRET_SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET="{client_secret}"''' ) )