Update testing structure for dbt Labs testing support - postgres

.github/workflows/ci_dbt_test_package.yml

@@ -0,0 +1,33 @@
+# **what?**
+# Run tests for dbt_artifacts against supported adapters
+
+# **why?**
+# To ensure that dbt_artifacts works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      # this just tests with postgres so no variables need to be passed through.
+      # When it's time to add more adapters you will need to pass through inputs for
+      # the other adapters as shown in the below example for redshift
+    #   with:
+    #     # redshift
+    #     REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+    #     REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+    #     REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+    #     REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
+    #     REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }}
+    #   secrets:
+    #     DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}

.github/workflows/ci_lint_package.yml

@@ -8,7 +8,7 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
+  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}

.github/workflows/ci_test_package.yml

@@ -8,7 +8,7 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
+  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
@@ -20,6 +20,13 @@ env:
   DBT_ENV_SECRET_DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
   DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
   DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
+  # These are not secrets but are used in the profiles.yml file
+  POSTGRES_HOST: "localhost"
+  POSTGRES_USER: "postgres"
+  DBT_ENV_SECRET_POSTGRES_PASS: "postgres"
+  POSTGRES_PORT: 5432
+  POSTGRES_DATABASE: "postgres"
+  POSTGRES_SCHEMA: "public"
   # Env var to test version
   LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_8_0 # A dbt version supported by both the last release and this one
   # Env vars to test invocations model
@@ -127,7 +134,10 @@ jobs:
       postgres:
         image: postgres
         env:
-          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }}
+          POSTGRES_DB: ${{ env.POSTGRES_DATABASE }}
+          POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/main_lint_package.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
+  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}

.github/workflows/main_test_package.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
+  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}

.github/workflows/publish_docs_on_release.yml

@@ -12,7 +12,7 @@ permissions:
 
 env:
   # These are configured in GitHub secrets
-  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
+  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests
   DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}

.gitignore

@@ -13,5 +13,6 @@ env.sh
 .python_version
 .vscode
 .venv
+env-*
 
 .DS_Store

CONTRIBUTING.md

@@ -58,11 +58,11 @@ Tox will take care of installing the dependencies for each environment, so you d
     pipx install tox
     ```
 
-3. Copy and paste the `integration_test_project/example-env.sh` file and save as `env.sh`. Fill in the missing values.
+3. Copy and paste the `integration_tests/example-env.sh` file and save as `env.sh`. Fill in the missing values.
    If you want to run only tests for a specific database, you can leave the other values blank.
 
     ```bash
-    cp integration_test_project/example-env.sh env.sh
+    cp integration_tests/example-env.sh env.sh
     vim env.sh
     ```
 
@@ -78,6 +78,7 @@ Tox will take care of installing the dependencies for each environment, so you d
     tox -e integration_snowflake # For the Snowflake tests
     tox -e integration_databricks # For the Databricks tests
     tox -e integration_bigquery # For the BigQuery tests
+    tox -e dbt_integration_postgres # for postgres tests - you will need to manually install dbt-postgres first
     ```
 
 The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac,

dbt_project.yml

@@ -2,7 +2,7 @@ name: "dbt_artifacts"
 version: "2.7.0"
 config-version: 2
 require-dbt-version: [">=1.3.0", "<1.9.0"]
-profile: "dbt_artifacts"
+profile: "integration_tests"
 
 clean-targets: # folders to be removed by `dbt clean`
   - "target"

integration_test_project/dbt_project.yml → integration_tests/dbt_project.yml

@@ -2,7 +2,7 @@ name: "artifacts_integration_tests"
 version: "1.0"
 config-version: 2
 
-profile: "dbt_artifacts"
+profile: "integration_tests"
 
 model-paths: ["models"]
 analysis-paths: ["analysis"]

integration_test_project/example-env.sh → integration_tests/example-env.sh

@@ -3,6 +3,14 @@
 export DBT_PROFILES_DIR=.
 export GITHUB_SHA=local_test # used for the schema name
 
+# These are the default values
+export POSTGRES_HOST=localhost
+export POSTGRES_USER=root
+export DBT_ENV_SECRET_POSTGRES_PASS=password
+export POSTGRES_PORT=5432
+export POSTGRES_DATABASE=postgres
+export POSTGRES_SCHEMA=public
+
 # Change these
 export DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT=
 export DBT_ENV_SECRET_SNOWFLAKE_TEST_USER=

integration_tests/package-lock.yml

@@ -0,0 +1,3 @@
+packages:
+  - local: ../
+sha1_hash: de2deba3d66ce03d8c02949013650cc9b94f6030

integration_test_project/profiles.yml → integration_tests/profiles.yml

@@ -5,7 +5,7 @@ config:
   send_anonymous_usage_stats: False
   use_colors: True
 
-dbt_artifacts:
+integration_tests:
   target: snowflake
   outputs:
     snowflake:
@@ -45,12 +45,12 @@ dbt_artifacts:
       retries: 1
     postgres:
       type: postgres
-      host: localhost
-      user: postgres
-      password: postgres
-      port: 5432
-      dbname: postgres
-      schema: public
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
       threads: 8
     sqlserver:
       type: sqlserver

supported_adapters.env

@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=postgres