Merge pull request #33 from emmyoop/er/support-dbt-testing-format

Update testing structure for dbt Labs testing support - postgres
get-select · Nov 13, 2024 · 9818fe6 · 9818fe6
2 parents 43400e3 + 0846d99
commit 9818fe6
Show file tree

Hide file tree

Showing 17 changed files with 92 additions and 36 deletions.
diff --git a/.github/workflows/ci_dbt_test_package.yml b/.github/workflows/ci_dbt_test_package.yml
@@ -0,0 +1,30 @@
+# **what?**
+# Run tests for dbt-snowflake-query-tags against supported adapters
+
+# **why?**
+# To ensure that dbt-snowflake-query-tags works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      with:
+        SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+        SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+        SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+        SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+        SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
+      secrets:
+        SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+        DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml
@@ -8,14 +8,15 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project
+  DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_tests
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
-  DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+  SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+  DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+  SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+  SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+  SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
 
 
 jobs:

diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml
@@ -9,7 +9,7 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project
+  DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_tests
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}

diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml
diff --git a/integration_test_project/dbt_project.yml → integration_tests/dbt_project.yml b/integration_test_project/dbt_project.yml → integration_tests/dbt_project.yml
@@ -2,7 +2,7 @@ name: 'dbt_snowflake_query_tags_tests'
 version: '1.0.0'
 config-version: 2
 
-profile: dbt_snowflake_query_tags_tests
+profile: integration_tests
 
 
 dispatch:

diff --git a/...project/models/materialized_ephemeral.sql → ...n_tests/models/materialized_ephemeral.sql b/...project/models/materialized_ephemeral.sql → ...n_tests/models/materialized_ephemeral.sql
diff --git a/...oject/models/materialized_incremental.sql → ...tests/models/materialized_incremental.sql b/...oject/models/materialized_incremental.sql → ...tests/models/materialized_incremental.sql
diff --git a/...est_project/models/materialized_table.sql → ...ation_tests/models/materialized_table.sql b/...est_project/models/materialized_table.sql → ...ation_tests/models/materialized_table.sql
diff --git a/...test_project/models/materialized_view.sql → ...ration_tests/models/materialized_view.sql b/...test_project/models/materialized_view.sql → ...ration_tests/models/materialized_view.sql
diff --git a/integration_test_project/packages.yml → integration_tests/packages.yml b/integration_test_project/packages.yml → integration_tests/packages.yml
diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml
@@ -0,0 +1,31 @@
+# HEY! This file is used in the dbt-snowflake-query-tags integrations tests with GitHub Actions.
+# You should __NEVER__ check credentials into version control. Thanks for reading :)
+
+config:
+    send_anonymous_usage_stats: False
+    use_colors: True
+
+integration_tests:
+  target: snowflake
+  outputs:
+    snowflake:
+      # type: snowflake
+      # account: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT') }}"
+      # user: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_USER') }}"
+      # password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD') }}"
+      # role: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE') }}"
+      # database: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE') }}"
+      # warehouse: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE') }}"
+      # schema: dbt_snowflake_query_tags_test_commit_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }}
+      # threads: 8
+
+      snowflake:
+        type: "snowflake"
+        account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+        user: "{{ env_var('SNOWFLAKE_USER') }}"
+        password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
+        role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+        database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+        warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+        schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
+        threads: 10
diff --git a/integration_test_project/seeds/test_seed.csv → integration_tests/seeds/test_seed.csv b/integration_test_project/seeds/test_seed.csv → integration_tests/seeds/test_seed.csv
diff --git a/...ation_test_project/snapshots/snapshot.sql → integration_tests/snapshots/snapshot.sql b/...ation_test_project/snapshots/snapshot.sql → integration_tests/snapshots/snapshot.sql
diff --git a/...ts/dbt_project_configured_meta/test_2.sql → ...ts/dbt_project_configured_meta/test_2.sql b/...ts/dbt_project_configured_meta/test_2.sql → ...ts/dbt_project_configured_meta/test_2.sql
diff --git a/integration_test_project/tests/test.sql → integration_tests/tests/test.sql b/integration_test_project/tests/test.sql → integration_tests/tests/test.sql
diff --git a/supported_adapters.env b/supported_adapters.env
@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=snowflake
diff --git a/tox.ini b/tox.ini
@@ -7,17 +7,30 @@ passenv =
     DBT_PROFILES_DIR
     GITHUB_SHA
     GITHUB_SHA_OVERRIDE
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_USER
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE
+    SNOWFLAKE_ACCOUNT
+    SNOWFLAKE_USER
+    DBT_ENV_SECRET_SNOWFLAKE_PASS
+    SNOWFLAKE_ROLE
+    SNOWFLAKE_DATABASE
+    SNOWFLAKE_WAREHOUSE
+    SNOWFLAKE_SCHEMA
 
 [testenv:snowflake]
-changedir = integration_test_project
+changedir = integration_tests
 deps = dbt-snowflake~=1.5.0
 commands =
     dbt deps
     dbt build --full-refresh
     dbt build
+
+# Snowflake integration tests for centralized dbt testing
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_snowflake]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
+skip_install = true
+commands =
+    dbt deps
+    dbt build --full-refresh --no-version-check
+    dbt build --no-version-check