diff --git a/.github/workflows/ci_dbt_test_package.yml b/.github/workflows/ci_dbt_test_package.yml new file mode 100644 index 00000000..e9959e1c --- /dev/null +++ b/.github/workflows/ci_dbt_test_package.yml @@ -0,0 +1,33 @@ +# **what?** +# Run tests for dbt_artifacts against supported adapters + +# **why?** +# To ensure that dbt_artifacts works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + # this just tests with postgres so no variables need to be passed through. + # When it's time to add more adapters you will need to pass through inputs for + # the other adapters as shown in the below example for redshift + # with: + # # redshift + # REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + # REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + # REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" + # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + # secrets: + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} \ No newline at end of file diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml index e2d33b7d..41fd8a0b 100644 --- a/.github/workflows/ci_lint_package.yml +++ b/.github/workflows/ci_lint_package.yml @@ -8,7 +8,7 @@ on: env: # These are configured in GitHub secrets - DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project + DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 5eb359fd..8c114247 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -8,7 +8,7 @@ on: env: # These are configured in GitHub secrets - DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project + DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} @@ -20,6 +20,13 @@ env: DBT_ENV_SECRET_DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }} + # These are not secrets but are used in the profiles.yml file + POSTGRES_HOST: "localhost" + POSTGRES_USER: "postgres" + DBT_ENV_SECRET_POSTGRES_PASS: "postgres" + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: "postgres" + POSTGRES_SCHEMA: "public" # Env var to test version LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_8_0 # A dbt version supported by both the last release and this one # Env vars to test invocations model @@ -127,7 +134,10 @@ jobs: postgres: image: postgres env: - POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_DB: ${{ env.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml index bd691804..79517da0 100644 --- a/.github/workflows/main_lint_package.yml +++ b/.github/workflows/main_lint_package.yml @@ -9,7 +9,7 @@ on: env: # These are configured in GitHub secrets - DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project + DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 1c55194e..bef714aa 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -9,7 +9,7 @@ on: env: # These are configured in GitHub secrets - DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project + DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml index 663486ff..be305e57 100644 --- a/.github/workflows/publish_docs_on_release.yml +++ b/.github/workflows/publish_docs_on_release.yml @@ -12,7 +12,7 @@ permissions: env: # These are configured in GitHub secrets - DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project + DBT_PROFILES_DIR: ${{ github.workspace }}/integration_tests DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} diff --git a/.gitignore b/.gitignore index eb7e5d18..c690a9d8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,6 @@ env.sh .python_version .vscode .venv +env-* .DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35653061..6a73d59d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,11 +58,11 @@ Tox will take care of installing the dependencies for each environment, so you d pipx install tox ``` -3. Copy and paste the `integration_test_project/example-env.sh` file and save as `env.sh`. Fill in the missing values. +3. Copy and paste the `integration_tests/example-env.sh` file and save as `env.sh`. Fill in the missing values. If you want to run only tests for a specific database, you can leave the other values blank. ```bash - cp integration_test_project/example-env.sh env.sh + cp integration_tests/example-env.sh env.sh vim env.sh ``` @@ -78,6 +78,7 @@ Tox will take care of installing the dependencies for each environment, so you d tox -e integration_snowflake # For the Snowflake tests tox -e integration_databricks # For the Databricks tests tox -e integration_bigquery # For the BigQuery tests + tox -e dbt_integration_postgres # for postgres tests - you will need to manually install dbt-postgres first ``` The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, diff --git a/dbt_project.yml b/dbt_project.yml index a02ad65a..de6c1aa6 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ name: "dbt_artifacts" version: "2.7.0" config-version: 2 require-dbt-version: [">=1.3.0", "<1.9.0"] -profile: "dbt_artifacts" +profile: "integration_tests" clean-targets: # folders to be removed by `dbt clean` - "target" diff --git a/integration_test_project/dbt_project.yml b/integration_tests/dbt_project.yml similarity index 96% rename from integration_test_project/dbt_project.yml rename to integration_tests/dbt_project.yml index aae6f8df..b5efc509 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -2,7 +2,7 @@ name: "artifacts_integration_tests" version: "1.0" config-version: 2 -profile: "dbt_artifacts" +profile: "integration_tests" model-paths: ["models"] analysis-paths: ["analysis"] diff --git a/integration_test_project/example-env.sh b/integration_tests/example-env.sh similarity index 84% rename from integration_test_project/example-env.sh rename to integration_tests/example-env.sh index 47cb0d67..421ff532 100755 --- a/integration_test_project/example-env.sh +++ b/integration_tests/example-env.sh @@ -3,6 +3,14 @@ export DBT_PROFILES_DIR=. export GITHUB_SHA=local_test # used for the schema name +# These are the default values +export POSTGRES_HOST=localhost +export POSTGRES_USER=root +export DBT_ENV_SECRET_POSTGRES_PASS=password +export POSTGRES_PORT=5432 +export POSTGRES_DATABASE=postgres +export POSTGRES_SCHEMA=public + # Change these export DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT= export DBT_ENV_SECRET_SNOWFLAKE_TEST_USER= diff --git a/integration_test_project/models/aliased.sql b/integration_tests/models/aliased.sql similarity index 100% rename from integration_test_project/models/aliased.sql rename to integration_tests/models/aliased.sql diff --git a/integration_test_project/models/exposure.md b/integration_tests/models/exposure.md similarity index 100% rename from integration_test_project/models/exposure.md rename to integration_tests/models/exposure.md diff --git a/integration_test_project/models/incremental.sql b/integration_tests/models/incremental.sql similarity index 100% rename from integration_test_project/models/incremental.sql rename to integration_tests/models/incremental.sql diff --git a/integration_test_project/models/non_incremental.sql b/integration_tests/models/non_incremental.sql similarity index 100% rename from integration_test_project/models/non_incremental.sql rename to integration_tests/models/non_incremental.sql diff --git a/integration_test_project/models/sources.yml b/integration_tests/models/sources.yml similarity index 100% rename from integration_test_project/models/sources.yml rename to integration_tests/models/sources.yml diff --git a/integration_test_project/models/tests_and_exposures.yml b/integration_tests/models/tests_and_exposures.yml similarity index 100% rename from integration_test_project/models/tests_and_exposures.yml rename to integration_tests/models/tests_and_exposures.yml diff --git a/integration_tests/package-lock.yml b/integration_tests/package-lock.yml new file mode 100644 index 00000000..10c1f1a7 --- /dev/null +++ b/integration_tests/package-lock.yml @@ -0,0 +1,3 @@ +packages: + - local: ../ +sha1_hash: de2deba3d66ce03d8c02949013650cc9b94f6030 diff --git a/integration_test_project/packages.yml b/integration_tests/packages.yml similarity index 100% rename from integration_test_project/packages.yml rename to integration_tests/packages.yml diff --git a/integration_test_project/profiles.yml b/integration_tests/profiles.yml similarity index 88% rename from integration_test_project/profiles.yml rename to integration_tests/profiles.yml index 33a1117a..6510b405 100644 --- a/integration_test_project/profiles.yml +++ b/integration_tests/profiles.yml @@ -5,7 +5,7 @@ config: send_anonymous_usage_stats: False use_colors: True -dbt_artifacts: +integration_tests: target: snowflake outputs: snowflake: @@ -45,12 +45,12 @@ dbt_artifacts: retries: 1 postgres: type: postgres - host: localhost - user: postgres - password: postgres - port: 5432 - dbname: postgres - schema: public + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 8 sqlserver: type: sqlserver diff --git a/integration_test_project/seeds/my_seed.csv b/integration_tests/seeds/my_seed.csv similarity index 100% rename from integration_test_project/seeds/my_seed.csv rename to integration_tests/seeds/my_seed.csv diff --git a/integration_test_project/snapshots/snapshot.sql b/integration_tests/snapshots/snapshot.sql similarity index 100% rename from integration_test_project/snapshots/snapshot.sql rename to integration_tests/snapshots/snapshot.sql diff --git a/integration_test_project/tests/singular_test.sql b/integration_tests/tests/singular_test.sql similarity index 100% rename from integration_test_project/tests/singular_test.sql rename to integration_tests/tests/singular_test.sql diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 00000000..79c9a199 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres diff --git a/tox.ini b/tox.ini index 93d870d2..794c93a2 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ capitalisation_policy = lower forbid_subquery_in = both [sqlfluff:templater:dbt] -profiles_dir = integration_test_project +profiles_dir = integration_tests [testenv] passenv = @@ -72,6 +72,12 @@ passenv = GITHUB_SHA_OVERRIDE GITHUB_SHA DBT_VERSION + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT DBT_ENV_SECRET_SNOWFLAKE_TEST_USER DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD @@ -114,11 +120,11 @@ commands = sqlfluff fix models --ignore parsing # Generate docs [testenv:generate_docs] deps = dbt-snowflake~=1.8.0 -commands = dbt docs generate --profiles-dir integration_test_project +commands = dbt docs generate --profiles-dir integration_tests # Snowflake integration tests [testenv:integration_snowflake] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.8.0 commands = dbt clean @@ -127,7 +133,7 @@ commands = [testenv:integration_snowflake_1_3_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.3.0 commands = dbt clean @@ -135,7 +141,7 @@ commands = dbt build --target snowflake [testenv:integration_snowflake_1_4_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.4.0 commands = dbt clean @@ -143,7 +149,7 @@ commands = dbt build --target snowflake [testenv:integration_snowflake_1_5_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.5.0 commands = dbt clean @@ -151,7 +157,7 @@ commands = dbt build --target snowflake [testenv:integration_snowflake_1_6_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.6.0 commands = dbt clean @@ -159,7 +165,7 @@ commands = dbt build --target snowflake [testenv:integration_snowflake_1_7_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.7.0 commands = dbt clean @@ -167,7 +173,7 @@ commands = dbt build --target snowflake [testenv:integration_snowflake_1_8_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-snowflake~=1.8.0 commands = dbt clean @@ -176,7 +182,7 @@ commands = # Databricks integration tests [testenv:integration_databricks] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.8.0 commands = dbt clean @@ -184,7 +190,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_3_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.3.0 commands = dbt clean @@ -192,7 +198,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_4_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.4.0 commands = dbt clean @@ -200,7 +206,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_5_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.5.0 commands = dbt clean @@ -208,7 +214,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_6_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.6.0 commands = dbt clean @@ -216,7 +222,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_7_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.7.0 commands = dbt clean @@ -224,7 +230,7 @@ commands = dbt build --target databricks [testenv:integration_databricks_1_8_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-databricks~=1.8.0 commands = dbt clean @@ -233,7 +239,7 @@ commands = # Bigquery integration tests [testenv:integration_bigquery] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.8.0 commands = dbt clean @@ -241,7 +247,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_3_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.3.0 commands = dbt clean @@ -249,7 +255,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_4_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.4.0 commands = dbt clean @@ -257,7 +263,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_5_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.5.0 commands = dbt clean @@ -265,7 +271,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_6_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.6.0 commands = dbt clean @@ -273,7 +279,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_7_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.7.0 commands = dbt clean @@ -281,7 +287,7 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_8_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-bigquery~=1.8.0 commands = dbt clean @@ -290,15 +296,16 @@ commands = # Spark integration test (disabled) [testenv:integration_spark] -changedir = integration_test_project +changedir = integration_tests deps = dbt-spark[ODBC]~=1.4.0 commands = dbt clean dbt deps dbt build --exclude snapshot --target spark +# Postgres integration test [testenv:integration_postgres] -changedir = integration_test_project +changedir = integration_tests deps = dbt-core~=1.8.0 dbt-postgres~=1.8.0 @@ -308,7 +315,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_3_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-postgres~=1.3.0 commands = dbt clean @@ -316,7 +323,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_4_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-postgres~=1.4.0 commands = dbt clean @@ -324,7 +331,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_5_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-postgres~=1.5.0 commands = dbt clean @@ -332,7 +339,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_6_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-postgres~=1.6.0 commands = dbt clean @@ -340,7 +347,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_7_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-postgres~=1.7.0 commands = dbt clean @@ -348,7 +355,7 @@ commands = dbt build --target postgres [testenv:integration_postgres_1_8_0] -changedir = integration_test_project +changedir = integration_tests deps = dbt-core~=1.8.0 dbt-postgres~=1.8.0 @@ -357,8 +364,21 @@ commands = dbt deps dbt build --target postgres +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt clean + dbt deps + dbt build --target postgres --no-version-check + +# sqlserver integration test [testenv:integration_sqlserver] -changedir = integration_test_project +changedir = integration_tests deps = dbt-sqlserver~=1.8.4 commands = dbt clean @@ -366,7 +386,7 @@ commands = dbt build --target sqlserver [testenv:integration_sqlserver_1_4_3] -changedir = integration_test_project +changedir = integration_tests deps = dbt-sqlserver~=1.4.3 commands = dbt clean @@ -374,7 +394,7 @@ commands = dbt build --target sqlserver [testenv:integration_sqlserver_1_7_4] -changedir = integration_test_project +changedir = integration_tests deps = dbt-sqlserver~=1.7.4 commands = dbt clean @@ -382,7 +402,7 @@ commands = dbt build --target sqlserver [testenv:integration_sqlserver_1_8_4] -changedir = integration_test_project +changedir = integration_tests deps = dbt-sqlserver~=1.8.4 commands = dbt clean