diff --git a/.changes/unreleased/Fixes-20240605-202614.yaml b/.changes/unreleased/Fixes-20240605-202614.yaml new file mode 100644 index 00000000..b7ab8eb0 --- /dev/null +++ b/.changes/unreleased/Fixes-20240605-202614.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Default to psycopg2-binary and allow overriding to psycopg2 via DBT_PSYCOPG2_NAME + (restores previous behavior) +time: 2024-06-05T20:26:14.801254-04:00 +custom: + Author: mikealfare + Issue: "96" diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh new file mode 100755 index 00000000..faee902c --- /dev/null +++ b/.github/scripts/psycopg2-check.sh @@ -0,0 +1,20 @@ +python -m venv venv +source venv/bin/activate +python -m pip install . + +if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then + if [[ $(pip show psycopg2-binary) ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION + fi +fi + +PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) +if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then + echo -e 'Expected: "$PSYCOPG2_EXPECTED_NAME" but found: "$PSYCOPG2_NAME"' + exit 1 +fi + +deactivate +rm -r ./venv diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1aefb7f5..d7367190 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,7 +45,7 @@ defaults: jobs: integration: name: Integration Tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -102,24 +102,41 @@ jobs: psycopg2-check: name: "Test psycopg2 build version" - runs-on: ${{ matrix.scenario.platform }} + runs-on: ${{ matrix.platform }} strategy: fail-fast: false matrix: - scenario: - - {platform: ubuntu-latest, psycopg2-name: psycopg2} - - {platform: macos-12, psycopg2-name: psycopg2-binary} + platform: [ubuntu-22.04, macos-12] + python-version: ["3.8", "3.11"] steps: - name: "Check out repository" uses: actions/checkout@v4 - - name: "Test psycopg2 name" - run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${{ matrix.scenario.psycopg2-name }}" ]]; then - exit 1 - fi + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: "Test psycopg2 name - default" + run: .github/scripts/psycopg2-check.sh + env: + PSYCOPG2_EXPECTED_NAME: psycopg2-binary + + - name: "Test psycopg2 name - invalid override" + run: .github/scripts/psycopg2-check.sh + env: + DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers + PSYCOPG2_EXPECTED_NAME: psycopg2-binary + + - name: "Test psycopg2 name - override" + run: .github/scripts/psycopg2-check.sh + env: + DBT_PSYCOPG2_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work + + - name: "Test psycopg2 name - manual override" + # verify that the workaround documented in the `README.md` continues to work + run: .github/scripts/psycopg2-check.sh + env: + PSYCOPG2_WORKAROUND: true + PSYCOPG2_EXPECTED_NAME: psycopg2 diff --git a/.gitignore b/.gitignore index 094ee4a9..b8d4accc 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,9 @@ cython_debug/ # testing artifacts /logs + +# MacOS +.DS_Store + +# vscode +.vscode/ diff --git a/README.md b/README.md index d5b8900a..285f5144 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,20 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) +### `psycopg2-binary` vs. `psycopg2` + +By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: + +```bash +if [[ $(pip show psycopg2-binary) ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION +fi +``` + +This ensures the version of `psycopg2` will match that of `psycopg2-binary`. + ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) diff --git a/pyproject.toml b/pyproject.toml index 0e93423c..2f97d5c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - # install `psycopg2` on linux (assumed production) - 'psycopg2>=2.9,<3.0; platform_system == "Linux"', - # install `psycopg2-binary` on macos/windows (assumed development) - 'psycopg2-binary>=2.9,<3.0; platform_system != "Linux"', + "psycopg2-binary>=2.9,<3.0", "dbt-adapters>=0.1.0a1,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0a1", diff --git a/tests/functional/configs/fixtures.py b/tests/functional/configs/fixtures.py deleted file mode 100644 index bb50393f..00000000 --- a/tests/functional/configs/fixtures.py +++ /dev/null @@ -1,201 +0,0 @@ -# NOTE: these fixtures also get used in `/tests/functional/saved_queries/` -import pytest - -models__schema_yml = """ -version: 2 -sources: - - name: raw - database: "{{ target.database }}" - schema: "{{ target.schema }}" - tables: - - name: 'seed' - identifier: "{{ var('seed_name', 'invalid') }}" - columns: - - name: id - data_tests: - - unique: - enabled: "{{ var('enabled_direct', None) | as_native }}" - - accepted_values: - enabled: "{{ var('enabled_direct', None) | as_native }}" - severity: "{{ var('severity_direct', None) | as_native }}" - values: [1,2] - -models: - - name: model - columns: - - name: id - data_tests: - - unique - - accepted_values: - values: [1,2,3,4] - -""" - -models__untagged_sql = """ -{{ - config(materialized='table') -}} - -select id, value from {{ source('raw', 'seed') }} - -""" - -models__tagged__model_sql = """ -{{ - config( - materialized='view', - tags=['tag_two'], - ) -}} - -{{ - config( - materialized='table', - tags=['tag_three'], - ) -}} - -select 4 as id, 2 as value - -""" - -seeds__seed_csv = """id,value -4,2 -""" - -tests__failing_sql = """ - -select 1 as fun - -""" - -tests__sleeper_agent_sql = """ -{{ config( - enabled = var('enabled_direct', False), - severity = var('severity_direct', 'WARN') -) }} - -select 1 as fun - -""" - -my_model = """ -select 1 as user -""" - -my_model_2 = """ -select * from {{ ref('my_model') }} -""" - -my_model_3 = """ -select * from {{ ref('my_model_2') }} -""" - -my_model_2_disabled = """ -{{ config(enabled=false) }} -select * from {{ ref('my_model') }} -""" - -my_model_3_disabled = """ -{{ config(enabled=false) }} -select * from {{ ref('my_model_2') }} -""" - -my_model_2_enabled = """ -{{ config(enabled=true) }} -select * from {{ ref('my_model') }} -""" - -my_model_3_enabled = """ -{{ config(enabled=true) }} -select * from {{ ref('my_model') }} -""" - -schema_all_disabled_yml = """ -version: 2 -models: - - name: my_model - - name: my_model_2 - config: - enabled: false - - name: my_model_3 - config: - enabled: false -""" - -schema_explicit_enabled_yml = """ -version: 2 -models: - - name: my_model - - name: my_model_2 - config: - enabled: true - - name: my_model_3 - config: - enabled: true -""" - -schema_partial_disabled_yml = """ -version: 2 -models: - - name: my_model - - name: my_model_2 - config: - enabled: false - - name: my_model_3 -""" - -schema_partial_enabled_yml = """ -version: 2 -models: - - name: my_model - - name: my_model_2 - config: - enabled: True - - name: my_model_3 -""" - -schema_invalid_enabled_yml = """ -version: 2 -models: - - name: my_model - config: - enabled: True and False - - name: my_model_3 -""" - -simple_snapshot = """{% snapshot mysnapshot %} - - {{ - config( - target_schema='snapshots', - strategy='timestamp', - unique_key='id', - updated_at='updated_at' - ) - }} - - select * from dummy - -{% endsnapshot %}""" - - -class BaseConfigProject: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "untagged.sql": models__untagged_sql, - "tagged": {"model.sql": models__tagged__model_sql}, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def tests(self): - return { - "failing.sql": tests__failing_sql, - "sleeper_agent.sql": tests__sleeper_agent_sql, - } diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py deleted file mode 100644 index d99b32c4..00000000 --- a/tests/functional/configs/test_configs.py +++ /dev/null @@ -1,141 +0,0 @@ -import os - -from dbt.exceptions import ParsingError -from dbt.tests.util import ( - check_relations_equal, - run_dbt, - update_config_file, - write_file, -) -from dbt_common.dataclass_schema import ValidationError -import pytest - -from tests.functional.configs.fixtures import ( - BaseConfigProject, - simple_snapshot, -) - - -class TestConfigs(BaseConfigProject): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "tagged": { - # the model configs will override this - "materialized": "invalid", - # the model configs will append to these - "tags": ["tag_one"], - } - }, - }, - "seeds": { - "quote_columns": False, - }, - } - - def test_config_layering( - self, - project, - ): - # run seed - results = run_dbt(["seed"]) - assert len(results) == 1 - - # test the project-level tag, and both config() call tags - assert len(run_dbt(["run", "--model", "tag:tag_one"])) == 1 - assert len(run_dbt(["run", "--model", "tag:tag_two"])) == 1 - assert len(run_dbt(["run", "--model", "tag:tag_three"])) == 1 - check_relations_equal(project.adapter, ["seed", "model"]) - - # make sure we overwrote the materialization properly - tables = project.get_tables_in_schema() - assert tables["model"] == "table" - - -# In addition to testing an alternative target-paths setting, it tests that -# the attribute is jinja rendered and that the context "modules" works. -class TestTargetConfigs(BaseConfigProject): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "target-path": "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}", - "seeds": { - "quote_columns": False, - }, - } - - def test_alternative_target_paths(self, project): - # chdir to a different directory to test creation of target directory under project_root - os.chdir(project.profiles_dir) - run_dbt(["seed"]) - - target_path = "" - for d in os.listdir(project.project_root): - if os.path.isdir(os.path.join(project.project_root, d)) and d.startswith("target_"): - target_path = d - assert os.path.exists(os.path.join(project.project_root, target_path, "manifest.json")) - - -class TestInvalidTestsMaterializationProj(object): - def test_tests_materialization_proj_config(self, project): - config_patch = {"data_tests": {"materialized": "table"}} - update_config_file(config_patch, project.project_root, "dbt_project.yml") - tests_dir = os.path.join(project.project_root, "tests") - write_file("select * from foo", tests_dir, "test.sql") - - with pytest.raises(ValidationError): - run_dbt() - - -class TestInvalidSeedsMaterializationProj(object): - def test_seeds_materialization_proj_config(self, project): - config_patch = {"seeds": {"materialized": "table"}} - update_config_file(config_patch, project.project_root, "dbt_project.yml") - - seeds_dir = os.path.join(project.project_root, "seeds") - write_file("id1, id2\n1, 2", seeds_dir, "seed.csv") - - with pytest.raises(ValidationError): - run_dbt() - - -class TestInvalidSeedsMaterializationSchema(object): - def test_seeds_materialization_schema_config(self, project): - seeds_dir = os.path.join(project.project_root, "seeds") - write_file( - "version: 2\nseeds:\n - name: myseed\n config:\n materialized: table", - seeds_dir, - "schema.yml", - ) - write_file("id1, id2\n1, 2", seeds_dir, "myseed.csv") - - with pytest.raises(ValidationError): - run_dbt() - - -class TestInvalidSnapshotsMaterializationProj(object): - def test_snapshots_materialization_proj_config(self, project): - config_patch = {"snapshots": {"materialized": "table"}} - update_config_file(config_patch, project.project_root, "dbt_project.yml") - - snapshots_dir = os.path.join(project.project_root, "snapshots") - write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") - - with pytest.raises(ParsingError): - run_dbt() - - -class TestInvalidSnapshotsMaterializationSchema(object): - def test_snapshots_materialization_schema_config(self, project): - snapshots_dir = os.path.join(project.project_root, "snapshots") - write_file( - "version: 2\nsnapshots:\n - name: mysnapshot\n config:\n materialized: table", - snapshots_dir, - "schema.yml", - ) - write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") - - with pytest.raises(ValidationError): - run_dbt() diff --git a/tests/functional/configs/test_configs_in_schema_files.py b/tests/functional/configs/test_configs_in_schema_files.py deleted file mode 100644 index c1f37049..00000000 --- a/tests/functional/configs/test_configs_in_schema_files.py +++ /dev/null @@ -1,256 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.tests.util import ( - check_relations_equal, - get_manifest, - run_dbt, - write_file, -) -from dbt_common.exceptions import CompilationError -import pytest - - -models_alt__schema_yml = """ -version: 2 -sources: - - name: raw - database: "{{ target.database }}" - schema: "{{ target.schema }}" - tables: - - name: 'some_seed' - columns: - - name: id - -models: - - name: model - description: "This is a model description" - config: - tags: ['tag_in_schema'] - meta: - owner: 'Julie Smith' - my_attr: "{{ var('my_var') }}" - materialized: view - - columns: - - name: id - data_tests: - - not_null: - meta: - owner: 'Simple Simon' - - unique: - config: - meta: - owner: 'John Doe' -""" - -models_alt__untagged_sql = """ -{{ - config(materialized='table') -}} - -select id, value from {{ source('raw', 'some_seed') }} -""" - -models_alt__tagged__model_sql = """ -{{ - config( - materialized='view', - tags=['tag_1_in_model'], - ) -}} - -{{ - config( - materialized='table', - tags=['tag_2_in_model'], - ) -}} - -select 4 as id, 2 as value -""" - -models_no_materialized__model_sql = """ -{{ - config( - tags=['tag_1_in_model'], - ) -}} - -{{ - config( - tags=['tag_2_in_model'], - ) -}} - -select 4 as id, 2 as value -""" - -seeds_alt__some_seed_csv = """id,value -4,2 -""" - -extra_alt__untagged_yml = """ -version: 2 - -models: - - name: untagged - description: "This is a model description" - meta: - owner: 'Somebody Else' - config: - meta: - owner: 'Julie Smith' -""" - -extra_alt__untagged2_yml = """ -version: 2 - -models: - - name: untagged - description: "This is a model description" - data_tests: - - not_null: - error_if: ">2" - config: - error_if: ">2" -""" - - -class TestSchemaFileConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models_alt__schema_yml, - "untagged.sql": models_alt__untagged_sql, - "tagged": {"model.sql": models_alt__tagged__model_sql}, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"some_seed.csv": seeds_alt__some_seed_csv} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+meta": { - "company": "NuMade", - }, - "test": { - "+meta": { - "project": "test", - }, - "tagged": { - "+meta": { - "team": "Core Team", - }, - "tags": ["tag_in_project"], - "model": { - "materialized": "table", - "+meta": { - "owner": "Julie Dent", - }, - }, - }, - }, - }, - "vars": { - "test": { - "my_var": "TESTING", - } - }, - "seeds": { - "quote_columns": False, - }, - } - - def test_config_layering( - self, - project, - ): - # run seed - assert len(run_dbt(["seed"])) == 1 - - # test the project-level tag, and both config() call tags - assert len(run_dbt(["run", "--model", "tag:tag_in_project"])) == 1 - assert len(run_dbt(["run", "--model", "tag:tag_1_in_model"])) == 1 - assert len(run_dbt(["run", "--model", "tag:tag_2_in_model"])) == 1 - assert len(run_dbt(["run", "--model", "tag:tag_in_schema"])) == 1 - - # Verify that model nodes have expected tags and meta - manifest = get_manifest(project.project_root) - model_id = "model.test.model" - model_node = manifest.nodes[model_id] - meta_expected = { - "company": "NuMade", - "project": "test", - "team": "Core Team", - "owner": "Julie Smith", - "my_attr": "TESTING", - } - assert model_node.meta == meta_expected - assert model_node.config.meta == meta_expected - model_tags = ["tag_1_in_model", "tag_2_in_model", "tag_in_project", "tag_in_schema"] - model_node_tags = model_node.tags.copy() - model_node_tags.sort() - assert model_node_tags == model_tags - model_node_config_tags = model_node.config.tags.copy() - model_node_config_tags.sort() - assert model_node_config_tags == model_tags - model_meta = { - "company": "NuMade", - "project": "test", - "team": "Core Team", - "owner": "Julie Smith", - "my_attr": "TESTING", - } - assert model_node.config.meta == model_meta - - # make sure we overwrote the materialization properly - tables = project.get_tables_in_schema() - assert tables["model"] == "table" - check_relations_equal(project.adapter, ["some_seed", "model"]) - - # Remove materialized config from model - write_file( - models_no_materialized__model_sql, - project.project_root, - "models", - "tagged", - "model.sql", - ) - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model_node = manifest.nodes[model_id] - - assert model_node.config.materialized == "view" - model_unrendered_config = { - "materialized": "view", - "meta": {"my_attr": "TESTING", "owner": "Julie Smith"}, - "tags": ["tag_1_in_model", "tag_2_in_model"], - } - assert model_node.unrendered_config == model_unrendered_config - - # look for test meta - schema_file_id = model_node.patch_path - schema_file = manifest.files[schema_file_id] - tests = schema_file.get_tests("models", "model") - assert tests[0] in manifest.nodes - test = manifest.nodes[tests[0]] - expected_meta = {"owner": "Simple Simon"} - assert test.config.meta == expected_meta - test = manifest.nodes[tests[1]] - expected_meta = {"owner": "John Doe"} - assert test.config.meta == expected_meta - - # copy a schema file with multiple metas - # shutil.copyfile('extra-alt/untagged.yml', 'models-alt/untagged.yml') - write_file(extra_alt__untagged_yml, project.project_root, "models", "untagged.yml") - with pytest.raises(ParsingError): - run_dbt(["run"]) - - # copy a schema file with config key in top-level of test and in config dict - # shutil.copyfile('extra-alt/untagged2.yml', 'models-alt/untagged.yml') - write_file(extra_alt__untagged2_yml, project.project_root, "models", "untagged.yml") - with pytest.raises(CompilationError): - run_dbt(["run"]) diff --git a/tests/functional/configs/test_contract_configs.py b/tests/functional/configs/test_contract_configs.py deleted file mode 100644 index 4be935e9..00000000 --- a/tests/functional/configs/test_contract_configs.py +++ /dev/null @@ -1,527 +0,0 @@ -import os - -from dbt.exceptions import ParsingError -from dbt.tests.util import get_artifact, get_manifest, write_file -from dbt_common.exceptions import ValidationError -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -my_model_sql = """ -{{ - config( - materialized = "table" - ) -}} - -select - 'blue' as color, - 1 as id, - cast('2019-01-01' as date) as date_day -""" - -my_model_contract_sql = """ -{{ - config( - materialized = "table", - contract = {"enforced": true} - ) -}} - -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -my_model_contract_disabled_sql = """ -{{ - config( - materialized = "table", - contract = {"enforced": false} - ) -}} - -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -my_incremental_model_sql = """ -{{ - config( - materialized = "incremental" - ) -}} - -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -my_view_model_sql = """ -{{ - config( - materialized = "view" - ) -}} - -select - 1 as id, - 'blue' as color, - cast('2019-01-01' as date) as date_day -""" - -my_model_python_error = """ -import holidays, s3fs - - -def model(dbt, _): - dbt.config( - materialized="table", - packages=["holidays", "s3fs"], # how to import python libraries in dbt's context - ) - df = dbt.ref("my_model") - df_describe = df.describe() # basic statistics profiling - return df_describe -""" - -model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - columns: - - name: id - quote: true - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: string - - name: date_day - data_type: date -""" - -model_schema_alias_types_false_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - alias_types: false - columns: - - name: id - quote: true - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: string - - name: date_day - data_type: date -""" - -model_schema_ignore_unsupported_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - columns: - - name: id - quote: true - data_type: integer - description: hello - constraints: - - type: not_null - warn_unsupported: False - - type: primary_key - warn_unsupported: False - - type: check - warn_unsupported: False - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: date -""" - -model_schema_errors_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: text - - name: date_day - - name: python_model - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: date -""" - -model_schema_blank_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true -""" - -model_schema_complete_datatypes_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id - quote: true - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: date -""" - -model_schema_incomplete_datatypes_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id - quote: true - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - data_tests: - - unique - - name: color - - name: date_day - data_type: date -""" - - -class TestModelLevelContractEnabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_yml, - } - - def test__model_contract_true(self, project): - run_dbt(["run"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - my_model_columns = model.columns - my_model_config = model.config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is True - - expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True)], quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={})}" - - assert expected_columns == str(my_model_columns) - - # compiled fields aren't in the manifest above because it only has parsed fields - manifest_json = get_artifact(project.project_root, "target", "manifest.json") - compiled_code = manifest_json["nodes"][model_id]["compiled_code"] - cleaned_code = " ".join(compiled_code.split()) - assert ( - "select 'blue' as color, 1 as id, cast('2019-01-01' as date) as date_day" - == cleaned_code - ) - - # set alias_types to false (should fail to compile) - write_file( - model_schema_alias_types_false_yml, - project.project_root, - "models", - "constraints_schema.yml", - ) - run_dbt(["run"], expect_pass=False) - - -class TestProjectContractEnabledConfigs: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"test": {"+contract": {"enforced": True}}}} - - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_complete_datatypes_yml, - } - - def test_defined_column_type(self, project): - run_dbt(["run"], expect_pass=True) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - assert contract_actual_config.enforced is True - - -class TestProjectContractEnabledConfigsError: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+contract": { - "enforced": True, - }, - } - } - } - - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_incomplete_datatypes_yml, - } - - def test_undefined_column_type(self, project): - _, log_output = run_dbt_and_capture(["run", "-s", "my_model"], expect_pass=False) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is True - - expected_compile_error = "Please ensure that the column name and data_type are defined within the YAML configuration for the ['color'] column(s)." - - assert expected_compile_error in log_output - - -class TestModelContractEnabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": my_model_contract_sql, "constraints_schema.yml": model_schema_yml} - - def test__model_contract(self, project): - run_dbt(["run"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - assert contract_actual_config.enforced is True - - -class TestModelContractEnabledConfigsMissingDataTypes: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_contract_sql, - "constraints_schema.yml": model_schema_incomplete_datatypes_yml, - } - - def test_undefined_column_type(self, project): - _, log_output = run_dbt_and_capture(["run", "-s", "my_model"], expect_pass=False) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is True - - expected_compile_error = "Please ensure that the column name and data_type are defined within the YAML configuration for the ['color'] column(s)." - - assert expected_compile_error in log_output - - -class TestModelLevelContractDisabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_contract_disabled_sql, - "constraints_schema.yml": model_schema_yml, - } - - def test__model_contract_false(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is False - - -class TestModelLevelContractErrorMessages: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_incremental_model_sql, - "constraints_schema.yml": model_schema_yml, - } - - def test__config_errors(self, project): - with pytest.raises(ValidationError) as err_info: - run_dbt(["run"], expect_pass=False) - - exc_str = " ".join(str(err_info.value).split()) - expected_materialization_error = "Invalid value for on_schema_change: ignore. Models materialized as incremental with contracts enabled must set on_schema_change to 'append_new_columns' or 'fail'" - assert expected_materialization_error in str(exc_str) - - -class TestModelLevelConstraintsErrorMessages: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.py": my_model_python_error, - "constraints_schema.yml": model_schema_yml, - } - - def test__config_errors(self, project): - with pytest.raises(ParsingError) as err_info: - run_dbt(["run"], expect_pass=False) - - exc_str = " ".join(str(err_info.value).split()) - expected_materialization_error = "Language Error: Expected 'sql' but found 'python'" - assert expected_materialization_error in str(exc_str) - # This is a compile time error and we won't get here because the materialization check is parse time - expected_empty_data_type_error = "Columns with `data_type` Blank/Null not allowed on contracted models. Columns Blank/Null: ['date_day']" - assert expected_empty_data_type_error not in str(exc_str) - - -class TestModelLevelConstraintsWarningMessages: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_view_model_sql, - "constraints_schema.yml": model_schema_yml, - } - - def test__config_warning(self, project): - _, log_output = run_dbt_and_capture(["run"]) - - expected_materialization_warning = ( - "Constraint types are not supported for view materializations" - ) - assert expected_materialization_warning in str(log_output) - - # change to not show warnings, message should not be in logs - models_dir = os.path.join(project.project_root, "models") - write_file(model_schema_ignore_unsupported_yml, models_dir, "constraints_schema.yml") - _, log_output = run_dbt_and_capture(["run"]) - - expected_materialization_warning = ( - "Constraint types are not supported for view materializations" - ) - assert expected_materialization_warning not in str(log_output) - - -class TestSchemaContractEnabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_blank_yml, - } - - def test__schema_error(self, project): - with pytest.raises(ParsingError) as err_info: - run_dbt(["parse"], expect_pass=False) - - exc_str = " ".join(str(err_info.value).split()) - schema_error_expected = "Constraints must be defined in a `yml` schema configuration file" - assert schema_error_expected in str(exc_str) - - -class TestPythonModelLevelContractErrorMessages: - @pytest.fixture(scope="class") - def models(self): - return { - "python_model.py": my_model_python_error, - "constraints_schema.yml": model_schema_errors_yml, - } - - def test__python_errors(self, project): - with pytest.raises(ParsingError) as err_info: - run_dbt(["parse"], expect_pass=False) - - exc_str = " ".join(str(err_info.value).split()) - expected_python_error = "Language Error: Expected 'sql' but found 'python'" - assert expected_python_error in exc_str - - -class TestModelContractMissingYAMLColumns: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_contract_sql, - } - - def test__missing_column_contract_error(self, project): - results = run_dbt(["run"], expect_pass=False) - expected_error = ( - "This model has an enforced contract, and its 'columns' specification is missing" - ) - assert expected_error in results[0].message diff --git a/tests/functional/configs/test_custom_node_colors_configs.py b/tests/functional/configs/test_custom_node_colors_configs.py deleted file mode 100644 index 8bd55bc5..00000000 --- a/tests/functional/configs/test_custom_node_colors_configs.py +++ /dev/null @@ -1,341 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt -from dbt_common.dataclass_schema import ValidationError -import pytest - - -CUSTOM_NODE_COLOR_MODEL_LEVEL = "red" -CUSTOM_NODE_COLOR_SCHEMA_LEVEL = "blue" -CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT = "#121212" -CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER = "purple" -CUSTOM_NODE_COLOR_INVALID_HEX = '"#xxx111"' -CUSTOM_NODE_COLOR_INVALID_NAME = "notacolor" - -# F strings are a pain here so replacing XXX with the config above instead -models__custom_node_color__model_sql = """ -{{ config(materialized='view', docs={'node_color': 'XXX'}) }} - -select 1 as id - -""".replace( - "XXX", CUSTOM_NODE_COLOR_MODEL_LEVEL -) - -models__non_custom_node_color__model_sql = """ -{{ config(materialized='view') }} - -select 1 as id - -""" - -models__show_docs_false__model_sql = """ -{{ config(materialized='view', docs={"show": True}) }} - -select 1 as id -""" - -models__custom_node_color__schema_yml = """ -version: 2 - -models: - - name: custom_color_model - description: "This is a model description" - config: - docs: - node_color: {} -""".format( - CUSTOM_NODE_COLOR_SCHEMA_LEVEL -) - - -models__non_custom_node_color__schema_yml = """ -version: 2 - -models: - - name: non_custom_color_model - description: "This is a model description" - config: - docs: - node_color: {} - show: True -""".format( - CUSTOM_NODE_COLOR_SCHEMA_LEVEL -) - -# To check that incorect configs are raising errors -models__non_custom_node_color_invalid_config_docs__schema_yml = """ -version: 2 - -models: - - name: non_custom_node_color - description: "This is a model description" - config: - docs: - node_color: {} - show: True -""".format( - CUSTOM_NODE_COLOR_INVALID_HEX -) - -models__non_custom_node_color_invalid_docs__schema_yml = """ -version: 2 - -models: - - name: non_custom_node_color - description: "This is a model description" - docs: - node_color: {} - show: True -""".format( - CUSTOM_NODE_COLOR_INVALID_NAME -) - -models__custom_node_color_invalid_hex__model_sql = """ -{{ config(materialized='view', docs={"show": True, "node_color": XXX }) }} - -select 1 as id -""".replace( - "XXX", CUSTOM_NODE_COLOR_INVALID_HEX -) - - -class BaseCustomNodeColorModelvsProject: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+docs": {"node_color": CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT, "show": False}, - "subdirectory": { - "+docs": { - "node_color": CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER, - "show": True, - }, - }, - } - } - } - - -# validation that model level node_color configs supercede dbt_project.yml -class TestModelLevelProjectColorConfigs(BaseCustomNodeColorModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return {"custom_color_model.sql": models__custom_node_color__model_sql} - - def test__model_override_project(self, project): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.custom_color_model" - my_model_config = manifest.nodes[model_id].config - my_model_docs = manifest.nodes[model_id].docs - - node_color_actual_config = my_model_config["docs"].node_color - show_actual_config = my_model_config["docs"].show - node_color_actual_docs = my_model_docs.node_color - show_actual_docs = my_model_docs.show - - # check node_color config is in the right spots for each model - assert node_color_actual_config == CUSTOM_NODE_COLOR_MODEL_LEVEL - assert node_color_actual_docs == CUSTOM_NODE_COLOR_MODEL_LEVEL - assert not show_actual_config - assert not show_actual_docs - - -# validation that model level node_color configs supercede schema.yml -class TestModelLevelSchemaColorConfigs(BaseCustomNodeColorModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return { - "custom_color_model.sql": models__custom_node_color__model_sql, - "custom_color_schema.yml": models__custom_node_color__schema_yml, - } - - def test__model_override_schema(self, project): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.custom_color_model" - my_model_config = manifest.nodes[model_id].config - my_model_docs = manifest.nodes[model_id].docs - - node_color_actual_config = my_model_config["docs"].node_color - show_actual_config = my_model_config["docs"].show - node_color_actual_docs = my_model_docs.node_color - show_actual_docs = my_model_docs.show - - # check node_color config is in the right spots for each model - assert node_color_actual_config == CUSTOM_NODE_COLOR_MODEL_LEVEL - assert node_color_actual_docs == CUSTOM_NODE_COLOR_MODEL_LEVEL - assert not show_actual_config - assert not show_actual_docs - - -# validation that node_color configured on subdirectories in dbt_project.yml supercedes project root -class TestSubdirectoryColorConfigs(BaseCustomNodeColorModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return { - "subdirectory": { - "non_custom_color_model_subdirectory.sql": models__non_custom_node_color__model_sql - } - } - - def test__project_folder_override_project_root(self, project): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.non_custom_color_model_subdirectory" - my_model_config = manifest.nodes[model_id].config - my_model_docs = manifest.nodes[model_id].docs - - node_color_actual_config = my_model_config["docs"].node_color - show_actual_config = my_model_config["docs"].show - node_color_actual_docs = my_model_docs.node_color - show_actual_docs = my_model_docs.show - - # check node_color config is in the right spots for each model - assert node_color_actual_config == CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER - assert node_color_actual_docs == CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER - # in this case show should be True since the dbt_project.yml overrides the root setting for /subdirectory - assert show_actual_config - assert show_actual_docs - - -# validation that node_color configured in schema.yml supercedes dbt_project.yml -class TestSchemaOverProjectColorConfigs(BaseCustomNodeColorModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return { - "non_custom_color_model.sql": models__non_custom_node_color__model_sql, - "non_custom_color_schema.yml": models__non_custom_node_color__schema_yml, - } - - def test__schema_override_project( - self, - project, - ): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - - model_id = "model.test.non_custom_color_model" - my_model_config = manifest.nodes[model_id].config - my_model_docs = manifest.nodes[model_id].docs - - node_color_actual_config = my_model_config["docs"].node_color - show_actual_config = my_model_config["docs"].show - node_color_actual_docs = my_model_docs.node_color - show_actual_docs = my_model_docs.show - - # check node_color config is in the right spots for each model - assert node_color_actual_config == CUSTOM_NODE_COLOR_SCHEMA_LEVEL - assert node_color_actual_docs == CUSTOM_NODE_COLOR_SCHEMA_LEVEL - # in this case show should be True since the schema.yml overrides the dbt_project.yml - assert show_actual_config - assert show_actual_docs - - -# validation that docs: show configured in model file supercedes dbt_project.yml -class TestModelOverProjectColorConfigs(BaseCustomNodeColorModelvsProject): - @pytest.fixture(scope="class") - def models(self): - return {"show_docs_override_model.sql": models__show_docs_false__model_sql} - - def test__model_show_overrides_dbt_project( - self, - project, - ): - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - - model_id = "model.test.show_docs_override_model" - my_model_config = manifest.nodes[model_id].config - my_model_docs = manifest.nodes[model_id].docs - - node_color_actual_config = my_model_config["docs"].node_color - show_actual_config = my_model_config["docs"].show - node_color_actual_docs = my_model_docs.node_color - show_actual_docs = my_model_docs.show - - # check node_color config is in the right spots for each model - assert node_color_actual_config == CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT - assert node_color_actual_docs == CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT - # in this case show should be True since the schema.yml overrides the dbt_project.yml - assert show_actual_config - assert show_actual_docs - - -# validation that an incorrect color in dbt_project.yml raises an exception -class TestCustomNodeColorIncorrectColorProject: - @pytest.fixture(scope="class") - def models(self): # noqa: F811 - return {"non_custom_node_color.sql": models__non_custom_node_color__model_sql} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": {"+docs": {"node_color": CUSTOM_NODE_COLOR_INVALID_NAME, "show": False}} - } - } - - def test__invalid_color_project( - self, - project, - ): - with pytest.raises(ValidationError): - run_dbt(["compile"]) - - -# validation that an incorrect color in the config block raises an exception -class TestCustomNodeColorIncorrectColorModelConfig: - @pytest.fixture(scope="class") - def models(self): - return { - "custom_node_color_invalid_hex.sql": models__custom_node_color_invalid_hex__model_sql - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"+docs": {"node_color": "blue", "show": False}}} - - def test__invalid_color_config_block( - self, - project, - ): - with pytest.raises(ValidationError): - run_dbt(["compile"]) - - -# validation that an incorrect color in the YML file raises an exception -class TestCustomNodeColorIncorrectColorNameYMLConfig: - @pytest.fixture(scope="class") - def models(self): - return { - "non_custom_node_color.sql": models__non_custom_node_color__model_sql, - "invalid_custom_color.yml": models__non_custom_node_color_invalid_docs__schema_yml, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"+docs": {"node_color": "blue", "show": False}}} - - def test__invalid_color_docs_not_under_config( - self, - project, - ): - with pytest.raises(ValidationError): - run_dbt(["compile"]) - - -class TestCustomNodeColorIncorrectColorHEXYMLConfig: - @pytest.fixture(scope="class") - def models(self): - return { - "non_custom_node_color.sql": models__non_custom_node_color__model_sql, - "invalid_custom_color.yml": models__non_custom_node_color_invalid_config_docs__schema_yml, - } - - def test__invalid_color_docs_under_config( - self, - project, - ): - with pytest.raises(ValidationError): - run_dbt(["compile"]) diff --git a/tests/functional/configs/test_disabled_configs.py b/tests/functional/configs/test_disabled_configs.py deleted file mode 100644 index a8af7d46..00000000 --- a/tests/functional/configs/test_disabled_configs.py +++ /dev/null @@ -1,90 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.configs.fixtures import BaseConfigProject - - -class TestDisabledConfigs(BaseConfigProject): - @pytest.fixture(scope="class") - def dbt_profile_data(self, unique_schema): - return { - "test": { - "outputs": { - "default": { - "type": "postgres", - # make sure you can do this and get an int out - "threads": "{{ (1 + 3) | as_number }}", - "host": "localhost", - "port": "{{ (5400 + 32) | as_number }}", - "user": "root", - "pass": "password", - "dbname": "dbt", - "schema": unique_schema, - }, - "disabled": { - "type": "postgres", - # make sure you can do this and get an int out - "threads": "{{ (1 + 3) | as_number }}", - "host": "localhost", - "port": "{{ (5400 + 32) | as_number }}", - "user": "root", - "pass": "password", - "dbname": "dbt", - "schema": unique_schema, - }, - }, - "target": "default", - }, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "enabled": "{{ (target.name == 'default' | as_bool) }}", - }, - }, - # set the `var` result in schema.yml to be 'seed', so that the - # `source` call can suceed. - "vars": { - "test": { - "seed_name": "seed", - } - }, - "seeds": { - "quote_columns": False, - "test": { - "seed": { - "enabled": "{{ (target.name == 'default') | as_bool }}", - }, - }, - }, - "data_tests": { - "test": { - "enabled": "{{ (target.name == 'default') | as_bool }}", - "severity": "WARN", - }, - }, - } - - def test_disable_seed_partial_parse(self, project): - run_dbt(["--partial-parse", "seed", "--target", "disabled"]) - run_dbt(["--partial-parse", "seed", "--target", "disabled"]) - - def test_conditional_model(self, project): - # no seeds/models - enabled should eval to False because of the target - results = run_dbt(["seed", "--target", "disabled"]) - assert len(results) == 0 - results = run_dbt(["run", "--target", "disabled"]) - assert len(results) == 0 - results = run_dbt(["test", "--target", "disabled"]) - assert len(results) == 0 - - # has seeds/models - enabled should eval to True because of the target - results = run_dbt(["seed"]) - assert len(results) == 1 - results = run_dbt(["run"]) - assert len(results) == 2 - results = run_dbt(["test"]) - assert len(results) == 5 diff --git a/tests/functional/configs/test_disabled_model.py b/tests/functional/configs/test_disabled_model.py deleted file mode 100644 index 8355d9bf..00000000 --- a/tests/functional/configs/test_disabled_model.py +++ /dev/null @@ -1,390 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.tests.util import get_manifest, run_dbt -from dbt_common.dataclass_schema import ValidationError -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.configs import fixtures - - -# ensure double disabled doesn't throw error when set at schema level -class TestSchemaDisabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_all_disabled_yml, - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - } - - def test_disabled_config(self, project): - run_dbt(["parse"]) - - -# ensure this throws a specific error that the model is disabled -class TestSchemaDisabledConfigsFailure: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_partial_disabled_yml, - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - } - - def test_disabled_config(self, project): - with pytest.raises(CompilationError) as exc: - run_dbt(["parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "which is disabled" - assert expected_msg in exc_str - - -# ensure double disabled doesn't throw error when set in model configs -class TestModelDisabledConfigs: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2_disabled, - "my_model_3.sql": fixtures.my_model_3_disabled, - } - - def test_disabled_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" not in manifest.nodes - assert "model.test.my_model_3" not in manifest.nodes - - assert "model.test.my_model_2" in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - -# ensure config set in project.yml can be overridden in yaml file -class TestOverrideProjectConfigsInYaml: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_partial_enabled_yml, - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "my_model_2": { - "enabled": False, - }, - "my_model_3": { - "enabled": False, - }, - }, - } - } - - def test_override_project_yaml_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" not in manifest.nodes - - assert "model.test.my_model_2" not in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - -# ensure config set in project.yml can be overridden in sql file -class TestOverrideProjectConfigsInSQL: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2_enabled, - "my_model_3.sql": fixtures.my_model_3, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "my_model_2": { - "enabled": False, - }, - "my_model_3": { - "enabled": False, - }, - }, - } - } - - def test_override_project_sql_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" not in manifest.nodes - - assert "model.test.my_model_2" not in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - -# ensure false config set in yaml file can be overridden in sql file -class TestOverrideFalseYAMLConfigsInSQL: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_all_disabled_yml, - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2_enabled, - "my_model_3.sql": fixtures.my_model_3, - } - - def test_override_yaml_sql_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" not in manifest.nodes - - assert "model.test.my_model_2" not in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - -# ensure true config set in yaml file can be overridden by false in sql file -class TestOverrideTrueYAMLConfigsInSQL: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_explicit_enabled_yml, - "my_model.sql": fixtures.my_model, - "my_model_2.sql": fixtures.my_model_2_enabled, - "my_model_3.sql": fixtures.my_model_3_disabled, - } - - def test_override_yaml_sql_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" not in manifest.nodes - - assert "model.test.my_model_2" not in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - -# ensure error when enabling in schema file when multiple nodes exist within disabled -class TestMultipleDisabledNodesForUniqueIDFailure: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_partial_enabled_yml, - "my_model.sql": fixtures.my_model, - "folder_1": { - "my_model_2.sql": fixtures.my_model_2_disabled, - "my_model_3.sql": fixtures.my_model_3_disabled, - }, - "folder_2": { - "my_model_2.sql": fixtures.my_model_2_disabled, - "my_model_3.sql": fixtures.my_model_3_disabled, - }, - "folder_3": { - "my_model_2.sql": fixtures.my_model_2_disabled, - "my_model_3.sql": fixtures.my_model_3_disabled, - }, - } - - def test_disabled_config(self, project): - with pytest.raises(ParsingError) as exc: - run_dbt(["parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "Found 3 matching disabled nodes for model 'my_model_2'" - assert expected_msg in exc_str - - -# ensure error when enabling in schema file when multiple nodes exist within disabled -class TestMultipleDisabledNodesSuccess: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": fixtures.my_model, - "folder_1": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - "folder_2": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "folder_1": { - "enabled": False, - }, - "folder_2": { - "enabled": True, - }, - }, - } - } - - def test_multiple_disabled_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" in manifest.nodes - - expected_file_path = "folder_2" - assert expected_file_path in manifest.nodes["model.test.my_model_2"].original_file_path - assert expected_file_path in manifest.nodes["model.test.my_model_3"].original_file_path - - assert "model.test.my_model_2" in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - expected_disabled_file_path = "folder_1" - assert ( - expected_disabled_file_path - in manifest.disabled["model.test.my_model_2"][0].original_file_path - ) - assert ( - expected_disabled_file_path - in manifest.disabled["model.test.my_model_3"][0].original_file_path - ) - - -# ensure overrides work when enabling in sql file when multiple nodes exist within disabled -class TestMultipleDisabledNodesOverrideModel: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": fixtures.my_model, - "folder_1": { - "my_model_2.sql": fixtures.my_model_2_enabled, - "my_model_3.sql": fixtures.my_model_3, - }, - "folder_2": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3_enabled, - }, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "folder_1": { - "enabled": False, - }, - "folder_2": { - "enabled": False, - }, - }, - } - } - - def test_multiple_disabled_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" in manifest.nodes - - expected_file_path_2 = "folder_1" - assert expected_file_path_2 in manifest.nodes["model.test.my_model_2"].original_file_path - expected_file_path_3 = "folder_2" - assert expected_file_path_3 in manifest.nodes["model.test.my_model_3"].original_file_path - - assert "model.test.my_model_2" in manifest.disabled - assert "model.test.my_model_3" in manifest.disabled - - expected_disabled_file_path_2 = "folder_2" - assert ( - expected_disabled_file_path_2 - in manifest.disabled["model.test.my_model_2"][0].original_file_path - ) - expected_disabled_file_path_3 = "folder_1" - assert ( - expected_disabled_file_path_3 - in manifest.disabled["model.test.my_model_3"][0].original_file_path - ) - - -# ensure everything lands where it should when disabling multiple nodes with the same unique id -class TestManyDisabledNodesSuccess: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": fixtures.my_model, - "folder_1": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - "folder_2": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - "folder_3": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - "folder_4": { - "my_model_2.sql": fixtures.my_model_2, - "my_model_3.sql": fixtures.my_model_3, - }, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "folder_1": { - "enabled": False, - }, - "folder_2": { - "enabled": True, - }, - "folder_3": { - "enabled": False, - }, - "folder_4": { - "enabled": False, - }, - }, - } - } - - def test_many_disabled_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert "model.test.my_model_2" in manifest.nodes - assert "model.test.my_model_3" in manifest.nodes - - expected_file_path = "folder_2" - assert expected_file_path in manifest.nodes["model.test.my_model_2"].original_file_path - assert expected_file_path in manifest.nodes["model.test.my_model_3"].original_file_path - - assert len(manifest.disabled["model.test.my_model_2"]) == 3 - assert len(manifest.disabled["model.test.my_model_3"]) == 3 - - -class TestInvalidEnabledConfig: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": fixtures.schema_invalid_enabled_yml, - "my_model.sql": fixtures.my_model, - } - - def test_invalis_config(self, project): - with pytest.raises(ValidationError) as exc: - run_dbt(["parse"]) - exc_str = " ".join(str(exc.value).split()) # flatten all whitespace - expected_msg = "'True and False' is not of type 'boolean'" - assert expected_msg in exc_str diff --git a/tests/functional/configs/test_dupe_paths.py b/tests/functional/configs/test_dupe_paths.py deleted file mode 100644 index b9a98d21..00000000 --- a/tests/functional/configs/test_dupe_paths.py +++ /dev/null @@ -1,74 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -my_model_sql = """ -select 1 as fun -""" - -seed_csv = """id,value -4,2 -""" - -somedoc_md = """ -{% docs somedoc %} -Testing, testing -{% enddocs %} -""" - -schema_yml = """ -version: 2 -models: - - name: my_model - description: testing model -""" - - -# Either a docs or a yml file is necessary to see the problem -# when two of the paths in 'all_source_paths' are the same -class TestDupeProjectPaths: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "seed.csv": seed_csv, - "somedoc.md": somedoc_md, - "schema.yml": schema_yml, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "model-paths": ["models"], - "seed-paths": ["models"], - } - - def test_config_with_dupe_paths(self, project, dbt_project_yml): - results = run_dbt(["seed"]) - assert len(results) == 1 - results = run_dbt(["run"]) - assert len(results) == 1 - - -class TestDupeStrippedProjectPaths: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "seed.csv": seed_csv, - "somedoc.md": somedoc_md, - "schema.yml": schema_yml, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "model-paths": ["models/"], - "seed-paths": ["models"], - } - - def test_config_with_dupe_paths(self, project, dbt_project_yml): - results = run_dbt(["seed"]) - assert len(results) == 1 - results = run_dbt(["run"]) - assert len(results) == 1 diff --git a/tests/functional/configs/test_get_default.py b/tests/functional/configs/test_get_default.py deleted file mode 100644 index 36d420e0..00000000 --- a/tests/functional/configs/test_get_default.py +++ /dev/null @@ -1,26 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -models_get__any_model_sql = """ --- models/any_model.sql -select {{ config.get('made_up_nonexistent_key', 'default_value') }} as col_value - -""" - - -class TestConfigGetDefault: - @pytest.fixture(scope="class") - def models(self): - return {"any_model.sql": models_get__any_model_sql} - - def test_config_with_get_default( - self, - project, - ): - # This test runs a model with a config.get(key, default) - # The default value is 'default_value' and causes an error - results = run_dbt(["run"], expect_pass=False) - assert len(results) == 1 - assert str(results[0].status) == "error" - assert 'column "default_value" does not exist' in results[0].message diff --git a/tests/functional/configs/test_grant_configs.py b/tests/functional/configs/test_grant_configs.py deleted file mode 100644 index 23b884a1..00000000 --- a/tests/functional/configs/test_grant_configs.py +++ /dev/null @@ -1,155 +0,0 @@ -from dbt.tests.util import ( - get_manifest, - run_dbt, - write_config_file, - write_file, -) -import pytest - - -dbt_project_yml = """ -models: - test: - my_model: - +grants: - my_select: ["reporter", "bi"] -""" - -append_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - +my_select: ["someone"] -""" - - -my_model_base_sql = """ -select 1 as fun -""" - - -my_model_clobber_sql = """ -{{ config(grants={'my_select': ['other_user']}) }} -select 1 as fun -""" - -my_model_extend_sql = """ -{{ config(grants={'+my_select': ['other_user']}) }} -select 1 as fun -""" - -my_model_extend_string_sql = """ -{{ config(grants={'+my_select': 'other_user'}) }} -select 1 as fun -""" - -my_model_extend_twice_sql = """ -{{ config(grants={'+my_select': ['other_user']}) }} -{{ config(grants={'+my_select': ['alt_user']}) }} -select 1 as fun -""" - - -class TestGrantConfigs: - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": my_model_base_sql} - - @pytest.fixture(scope="class") - def project_config_update(self): - return dbt_project_yml - - def test_model_grant_config(self, project, logs_dir): - # This test uses "my_select" instead of "select", so we need - # use "parse" instead of "run" because we will get compilation - # errors for the grants. - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - assert model_id in manifest.nodes - - model = manifest.nodes[model_id] - model_config = model.config - assert hasattr(model_config, "grants") - - # no schema grant, no model grant, just project - expected = {"my_select": ["reporter", "bi"]} - assert model_config.grants == expected - - # add model grant with clobber - write_file(my_model_clobber_sql, project.project_root, "models", "my_model.sql") - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["other_user"]} - assert model_config.grants == expected - - # change model to extend grants - write_file(my_model_extend_sql, project.project_root, "models", "my_model.sql") - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["reporter", "bi", "other_user"]} - assert model_config.grants == expected - - # add schema file with extend - write_file(append_schema_yml, project.project_root, "models", "schema.yml") - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["reporter", "bi", "someone", "other_user"]} - assert model_config.grants == expected - - # change model file to have string instead of list - write_file(my_model_extend_string_sql, project.project_root, "models", "my_model.sql") - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["reporter", "bi", "someone", "other_user"]} - assert model_config.grants == expected - - # change model file to have string instead of list - write_file(my_model_extend_twice_sql, project.project_root, "models", "my_model.sql") - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["reporter", "bi", "someone", "other_user", "alt_user"]} - assert model_config.grants == expected - - # Remove grant from dbt_project - config = { - "config-version": 2, - "name": "test", - "version": "0.1.0", - "profile": "test", - "log-path": logs_dir, - } - write_config_file(config, project.project_root, "dbt_project.yml") - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["someone", "other_user", "alt_user"]} - assert model_config.grants == expected - - # Remove my_model config, leaving only schema file - write_file(my_model_base_sql, project.project_root, "models", "my_model.sql") - run_dbt(["parse"]) - - manifest = get_manifest(project.project_root) - model_config = manifest.nodes[model_id].config - - expected = {"my_select": ["someone"]} - assert model_config.grants == expected diff --git a/tests/functional/configs/test_indiv_tests.py b/tests/functional/configs/test_indiv_tests.py deleted file mode 100644 index 1084760a..00000000 --- a/tests/functional/configs/test_indiv_tests.py +++ /dev/null @@ -1,58 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.configs.fixtures import BaseConfigProject - - -class TestConfigIndivTests(BaseConfigProject): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "quote_columns": False, - }, - "vars": { - "test": { - "seed_name": "seed", - } - }, - "data_tests": {"test": {"enabled": True, "severity": "WARN"}}, - } - - def test_configuring_individual_tests( - self, - project, - ): - assert len(run_dbt(["seed"])) == 1 - assert len(run_dbt(["run"])) == 2 - - # all tests on (minus sleeper_agent) + WARN - assert len(run_dbt(["test"])) == 5 - - # turn off two of them directly - assert len(run_dbt(["test", "--vars", '{"enabled_direct": False}'])) == 3 - - # turn on sleeper_agent data test directly - assert ( - len( - run_dbt( - ["test", "--models", "sleeper_agent", "--vars", '{"enabled_direct": True}'] - ) - ) - == 1 - ) - - # set three to ERROR directly - results = run_dbt( - [ - "test", - "--models", - "config.severity:error", - "--vars", - '{"enabled_direct": True, "severity_direct": "ERROR"}', - ], - expect_pass=False, - ) - assert len(results) == 2 - assert results[0].status == "fail" - assert results[1].status == "fail" diff --git a/tests/functional/configs/test_unused_configs.py b/tests/functional/configs/test_unused_configs.py deleted file mode 100644 index a01ebc01..00000000 --- a/tests/functional/configs/test_unused_configs.py +++ /dev/null @@ -1,52 +0,0 @@ -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - - -seeds__seed_csv = """id,value -4,2 -""" - - -class TestUnusedModelConfigs: - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "test-paths": ["does-not-exist"], - "models": { - "test": { - "enabled": True, - } - }, - "seeds": { - "quote_columns": False, - }, - "sources": { - "test": { - "enabled": True, - } - }, - "data_tests": { - "test": { - "enabled": True, - } - }, - } - - def test_warn_unused_configuration_paths( - self, - project, - ): - with pytest.raises(CompilationError) as excinfo: - run_dbt(["--warn-error", "seed"]) - - assert "Configuration paths exist" in str(excinfo.value) - assert "- sources.test" in str(excinfo.value) - assert "- models.test" in str(excinfo.value) - assert "- models.test" in str(excinfo.value) - - run_dbt(["seed"]) diff --git a/tests/functional/simple_snapshot/data/invalidate_postgres.sql b/tests/functional/simple_snapshot/data/invalidate_postgres.sql deleted file mode 100644 index b0bef3c6..00000000 --- a/tests/functional/simple_snapshot/data/invalidate_postgres.sql +++ /dev/null @@ -1,27 +0,0 @@ - --- update records 11 - 21. Change email and updated_at field -update {schema}.seed set - updated_at = updated_at + interval '1 hour', - email = case when id = 20 then 'pfoxj@creativecommons.org' else 'new_' || email end -where id >= 10 and id <= 20; - - --- invalidate records 11 - 21 -update {schema}.snapshot_expected set - dbt_valid_to = updated_at + interval '1 hour' -where id >= 10 and id <= 20; - - -update {schema}.snapshot_castillo_expected set - dbt_valid_to = "1-updated_at" + interval '1 hour' -where id >= 10 and id <= 20; - - -update {schema}.snapshot_alvarez_expected set - dbt_valid_to = updated_at + interval '1 hour' -where id >= 10 and id <= 20; - - -update {schema}.snapshot_kelly_expected set - dbt_valid_to = updated_at + interval '1 hour' -where id >= 10 and id <= 20; diff --git a/tests/functional/simple_snapshot/data/seed_pg.sql b/tests/functional/simple_snapshot/data/seed_pg.sql deleted file mode 100644 index a22a2359..00000000 --- a/tests/functional/simple_snapshot/data/seed_pg.sql +++ /dev/null @@ -1,223 +0,0 @@ - create table {database}.{schema}.seed ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - updated_at TIMESTAMP WITHOUT TIME ZONE -); - -create table {database}.{schema}.snapshot_expected ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - - -- snapshotting fields - updated_at TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, - dbt_scd_id TEXT, - dbt_updated_at TIMESTAMP WITHOUT TIME ZONE -); - - --- seed inserts --- use the same email for two users to verify that duplicated check_cols values --- are handled appropriately -insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values -(1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), -(2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), -(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), -(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), -(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), -(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), -(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), -(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), -(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), -(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), -(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), -(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), -(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), -(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), -(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), -(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), -(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), -(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), -(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), -(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); - - --- populate snapshot table -insert into {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed; - - - -create table {database}.{schema}.snapshot_castillo_expected ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - - -- snapshotting fields - "1-updated_at" TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, - dbt_scd_id TEXT, - dbt_updated_at TIMESTAMP WITHOUT TIME ZONE -); - --- one entry -insert into {database}.{schema}.snapshot_castillo_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - "1-updated_at", - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed where last_name = 'Castillo'; - -create table {database}.{schema}.snapshot_alvarez_expected ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - - -- snapshotting fields - updated_at TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, - dbt_scd_id TEXT, - dbt_updated_at TIMESTAMP WITHOUT TIME ZONE -); - --- 0 entries -insert into {database}.{schema}.snapshot_alvarez_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed where last_name = 'Alvarez'; - -create table {database}.{schema}.snapshot_kelly_expected ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - - -- snapshotting fields - updated_at TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, - dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, - dbt_scd_id TEXT, - dbt_updated_at TIMESTAMP WITHOUT TIME ZONE -); - - --- 2 entries -insert into {database}.{schema}.snapshot_kelly_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed where last_name = 'Kelly'; diff --git a/tests/functional/simple_snapshot/data/shared_macros.sql b/tests/functional/simple_snapshot/data/shared_macros.sql deleted file mode 100644 index 9bdfdd26..00000000 --- a/tests/functional/simple_snapshot/data/shared_macros.sql +++ /dev/null @@ -1,80 +0,0 @@ -{% macro get_snapshot_unique_id() -%} - {{ return(adapter.dispatch('get_snapshot_unique_id')()) }} -{%- endmacro %} - -{% macro default__get_snapshot_unique_id() -%} - {% do return("id || '-' || first_name") %} -{%- endmacro %} - -{# - mostly copy+pasted from dbt_utils, but I removed some parameters and added - a query that calls get_snapshot_unique_id -#} -{% test mutually_exclusive_ranges(model) %} - -with base as ( - select {{ get_snapshot_unique_id() }} as dbt_unique_id, - * - from {{ model }} -), -window_functions as ( - - select - dbt_valid_from as lower_bound, - coalesce(dbt_valid_to, '2099-1-1T00:00:01') as upper_bound, - - lead(dbt_valid_from) over ( - partition by dbt_unique_id - order by dbt_valid_from - ) as next_lower_bound, - - row_number() over ( - partition by dbt_unique_id - order by dbt_valid_from desc - ) = 1 as is_last_record - - from base - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound < upper_bound, - is_last_record - ) as lower_bound_less_than_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound = next_lower_bound, - is_last_record, - false - ) as upper_bound_equal_to_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_less_than_upper_bound - and upper_bound_equal_to_next_lower_bound - ) -) - -select * from validation_errors -{% endtest %} diff --git a/tests/functional/simple_snapshot/data/update.sql b/tests/functional/simple_snapshot/data/update.sql deleted file mode 100644 index 890959f3..00000000 --- a/tests/functional/simple_snapshot/data/update.sql +++ /dev/null @@ -1,261 +0,0 @@ --- insert v2 of the 11 - 21 records - -insert into {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id >= 10 and id <= 20; - - -insert into {database}.{schema}.snapshot_castillo_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - "1-updated_at", - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id >= 10 and id <= 20 and last_name = 'Castillo'; - - -insert into {database}.{schema}.snapshot_alvarez_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id >= 10 and id <= 20 and last_name = 'Alvarez'; - - -insert into {database}.{schema}.snapshot_kelly_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id >= 10 and id <= 20 and last_name = 'Kelly'; - --- insert 10 new records -insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values -(21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), -(22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), -(23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), -(24, 'William', 'Watkins', 'wwatkinsn@guardian.co.uk', 'Male', '78.155.84.6', '2016-03-08 19:13:08'), -(25, 'Judy', 'Cooper', 'jcoopero@google.com.au', 'Female', '24.149.123.184', '2016-10-05 20:49:33'), -(26, 'Shirley', 'Castillo', 'scastillop@samsung.com', 'Female', '129.252.181.12', '2016-06-20 21:12:21'), -(27, 'Justin', 'Harper', 'jharperq@opera.com', 'Male', '131.172.103.218', '2016-05-21 22:56:46'), -(28, 'Marie', 'Medina', 'mmedinar@nhs.uk', 'Female', '188.119.125.67', '2015-10-08 13:44:33'), -(29, 'Kelly', 'Edwards', 'kedwardss@phoca.cz', 'Female', '47.121.157.66', '2015-09-15 06:33:37'), -(30, 'Carl', 'Coleman', 'ccolemant@wikipedia.org', 'Male', '82.227.154.83', '2016-05-26 16:46:40'); - - --- add these new records to the snapshot table -insert into {database}.{schema}.snapshot_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id > 20; - - --- add these new records to the snapshot table -insert into {database}.{schema}.snapshot_castillo_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - "1-updated_at", - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id > 20 and last_name = 'Castillo'; - -insert into {database}.{schema}.snapshot_alvarez_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id > 20 and last_name = 'Alvarez'; - -insert into {database}.{schema}.snapshot_kelly_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - dbt_valid_from, - dbt_valid_to, - dbt_updated_at, - dbt_scd_id -) - -select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - -- fields added by snapshotting - updated_at as dbt_valid_from, - null::timestamp as dbt_valid_to, - updated_at as dbt_updated_at, - md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id -from {database}.{schema}.seed -where id > 20 and last_name = 'Kelly'; diff --git a/tests/functional/simple_snapshot/fixtures.py b/tests/functional/simple_snapshot/fixtures.py deleted file mode 100644 index 04e4905d..00000000 --- a/tests/functional/simple_snapshot/fixtures.py +++ /dev/null @@ -1,389 +0,0 @@ -snapshots_select__snapshot_sql = """ -{% snapshot snapshot_castillo %} - - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='"1-updated_at"', - ) - }} - select id,first_name,last_name,email,gender,ip_address,updated_at as "1-updated_at" from {{target.database}}.{{schema}}.seed where last_name = 'Castillo' - -{% endsnapshot %} - -{% snapshot snapshot_alvarez %} - - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{schema}}.seed where last_name = 'Alvarez' - -{% endsnapshot %} - - -{% snapshot snapshot_kelly %} - {# This has no target_database set, which is allowed! #} - {{ - config( - target_schema=schema, - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{schema}}.seed where last_name = 'Kelly' - -{% endsnapshot %} -""" - -snapshots_pg_custom__snapshot_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - target_schema=var('target_schema', schema), - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='custom', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} -""" - - -macros_custom_snapshot__custom_sql = """ -{# A "custom" strategy that's really just the timestamp one #} -{% macro snapshot_custom_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} - {% set primary_key = config['unique_key'] %} - {% set updated_at = config['updated_at'] %} - - {% set row_changed_expr -%} - ({{ snapshotted_rel }}.{{ updated_at }} < {{ current_rel }}.{{ updated_at }}) - {%- endset %} - - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} - - {% do return({ - "unique_key": primary_key, - "updated_at": updated_at, - "row_changed": row_changed_expr, - "scd_id": scd_id_expr - }) %} -{% endmacro %} -""" - - -models__schema_yml = """ -version: 2 -snapshots: - - name: snapshot_actual - data_tests: - - mutually_exclusive_ranges - config: - meta: - owner: 'a_owner' -""" - -models__schema_with_target_schema_yml = """ -version: 2 -snapshots: - - name: snapshot_actual - data_tests: - - mutually_exclusive_ranges - config: - meta: - owner: 'a_owner' - target_schema: schema_from_schema_yml -""" - -models__ref_snapshot_sql = """ -select * from {{ ref('snapshot_actual') }} -""" - -macros__test_no_overlaps_sql = """ -{% macro get_snapshot_unique_id() -%} - {{ return(adapter.dispatch('get_snapshot_unique_id')()) }} -{%- endmacro %} - -{% macro default__get_snapshot_unique_id() -%} - {% do return("id || '-' || first_name") %} -{%- endmacro %} - -{# - mostly copy+pasted from dbt_utils, but I removed some parameters and added - a query that calls get_snapshot_unique_id -#} -{% test mutually_exclusive_ranges(model) %} - -with base as ( - select {{ get_snapshot_unique_id() }} as dbt_unique_id, - * - from {{ model }} -), -window_functions as ( - - select - dbt_valid_from as lower_bound, - coalesce(dbt_valid_to, '2099-1-1T00:00:01') as upper_bound, - - lead(dbt_valid_from) over ( - partition by dbt_unique_id - order by dbt_valid_from - ) as next_lower_bound, - - row_number() over ( - partition by dbt_unique_id - order by dbt_valid_from desc - ) = 1 as is_last_record - - from base - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound < upper_bound, - is_last_record - ) as lower_bound_less_than_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound = next_lower_bound, - is_last_record, - false - ) as upper_bound_equal_to_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_less_than_upper_bound - and upper_bound_equal_to_next_lower_bound - ) -) - -select * from validation_errors -{% endtest %} -""" - - -snapshots_select_noconfig__snapshot_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - target_schema=var('target_schema', schema), - ) - }} - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} - -{% snapshot snapshot_castillo %} - - {{ - config( - target_database=var('target_database', database), - updated_at='"1-updated_at"', - ) - }} - select id,first_name,last_name,email,gender,ip_address,updated_at as "1-updated_at" from {{target.database}}.{{schema}}.seed where last_name = 'Castillo' - -{% endsnapshot %} - -{% snapshot snapshot_alvarez %} - - {{ - config( - target_database=var('target_database', database), - ) - }} - select * from {{target.database}}.{{schema}}.seed where last_name = 'Alvarez' - -{% endsnapshot %} - - -{% snapshot snapshot_kelly %} - {# This has no target_database set, which is allowed! #} - select * from {{target.database}}.{{schema}}.seed where last_name = 'Kelly' - -{% endsnapshot %} -""" - - -seeds__seed_newcol_csv = """id,first_name,last_name -1,Judith,Kennedy -2,Arthur,Kelly -3,Rachel,Moreno -""" - -seeds__seed_csv = """id,first_name -1,Judith -2,Arthur -3,Rachel -""" - - -snapshots_pg_custom_namespaced__snapshot_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - target_schema=var('target_schema', schema), - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='test.custom', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} -""" - -snapshots_pg__snapshot_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - target_schema=var('target_schema', schema), - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='updated_at', - ) - }} - - {% if var('invalidate_hard_deletes', 'false') | as_bool %} - {{ config(invalidate_hard_deletes=True) }} - {% endif %} - - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} -""" - -snapshots_pg__snapshot_no_target_schema_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='updated_at', - ) - }} - - {% if var('invalidate_hard_deletes', 'false') | as_bool %} - {{ config(invalidate_hard_deletes=True) }} - {% endif %} - - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} -""" - -models_slow__gen_sql = """ - -{{ config(materialized='ephemeral') }} - - -/* - Generates 50 rows that "appear" to update every - second to a query-er. - - 1 2020-04-21 20:44:00-04 0 - 2 2020-04-21 20:43:59-04 59 - 3 2020-04-21 20:43:58-04 58 - 4 2020-04-21 20:43:57-04 57 - - .... 1 second later .... - - 1 2020-04-21 20:44:01-04 1 - 2 2020-04-21 20:44:00-04 0 - 3 2020-04-21 20:43:59-04 59 - 4 2020-04-21 20:43:58-04 58 - - This view uses pg_sleep(2) to make queries against - the view take a non-trivial amount of time - - Use statement_timestamp() as it changes during a transactions. - If we used now() or current_time or similar, then the timestamp - of the start of the transaction would be returned instead. -*/ - -with gen as ( - - select - id, - date_trunc('second', statement_timestamp()) - (interval '1 second' * id) as updated_at - - from generate_series(1, 10) id - -) - -select - id, - updated_at, - extract(seconds from updated_at)::int as seconds - -from gen, pg_sleep(2) -""" - -snapshots_longtext__snapshot_sql = """ -{% snapshot snapshot_actual %} - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id', - strategy='timestamp', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{schema}}.super_long -{% endsnapshot %} -""" - -snapshots_check_col_noconfig__snapshot_sql = """ -{% snapshot snapshot_actual %} - select * from {{target.database}}.{{schema}}.seed -{% endsnapshot %} - -{# This should be exactly the same #} -{% snapshot snapshot_checkall %} - {{ config(check_cols='all') }} - select * from {{target.database}}.{{schema}}.seed -{% endsnapshot %} -""" diff --git a/tests/functional/simple_snapshot/test_basic_snapshot.py b/tests/functional/simple_snapshot/test_basic_snapshot.py deleted file mode 100644 index 6165e8e1..00000000 --- a/tests/functional/simple_snapshot/test_basic_snapshot.py +++ /dev/null @@ -1,373 +0,0 @@ -from datetime import datetime -import os - -from dbt.tests.util import ( - check_relations_equal, - relation_from_name, - run_dbt, - write_file, -) -import pytest -import pytz - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - macros_custom_snapshot__custom_sql, - models__ref_snapshot_sql, - models__schema_with_target_schema_yml, - models__schema_yml, - seeds__seed_csv, - seeds__seed_newcol_csv, - snapshots_pg__snapshot_no_target_schema_sql, - snapshots_pg__snapshot_sql, - snapshots_pg_custom__snapshot_sql, - snapshots_pg_custom_namespaced__snapshot_sql, -) - - -snapshots_check_col__snapshot_sql = """ -{% snapshot snapshot_actual %} - - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='check', - check_cols=['email'], - ) - }} - select * from {{target.database}}.{{schema}}.seed - -{% endsnapshot %} - -{# This should be exactly the same #} -{% snapshot snapshot_checkall %} - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='check', - check_cols='all', - ) - }} - select * from {{target.database}}.{{schema}}.seed -{% endsnapshot %} -""" - - -snapshots_check_col_noconfig__snapshot_sql = """ -{% snapshot snapshot_actual %} - select * from {{target.database}}.{{schema}}.seed -{% endsnapshot %} - -{# This should be exactly the same #} -{% snapshot snapshot_checkall %} - {{ config(check_cols='all') }} - select * from {{target.database}}.{{schema}}.seed -{% endsnapshot %} -""" - - -def snapshot_setup(project, num_snapshot_models=1): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - results = run_dbt(["snapshot"]) - assert len(results) == num_snapshot_models - - run_dbt(["test"]) - check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) - - path = os.path.join(project.test_data_dir, "invalidate_postgres.sql") - project.run_sql_file(path) - - path = os.path.join(project.test_data_dir, "update.sql") - project.run_sql_file(path) - - results = run_dbt(["snapshot"]) - assert len(results) == num_snapshot_models - - run_dbt(["test"]) - check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) - - -def ref_setup(project, num_snapshot_models=1): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - results = run_dbt(["snapshot"]) - assert len(results) == num_snapshot_models - - results = run_dbt(["run"]) - assert len(results) == 1 - - -class Basic: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_pg__snapshot_sql} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -class TestBasicSnapshot(Basic): - def test_basic_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=1) - - -class TestBasicRef(Basic): - def test_basic_ref(self, project): - ref_setup(project, num_snapshot_models=1) - - -class TestBasicTargetSchemaConfig(Basic): - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_pg__snapshot_no_target_schema_sql} - - @pytest.fixture(scope="class") - def project_config_update(self, unique_schema): - return { - "snapshots": { - "test": { - "target_schema": unique_schema + "_alt", - } - } - } - - def test_target_schema(self, project): - manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 5 - # ensure that the schema in the snapshot node is the same as target_schema - snapshot_id = "snapshot.test.snapshot_actual" - snapshot_node = manifest.nodes[snapshot_id] - assert snapshot_node.schema == f"{project.test_schema}_alt" - assert ( - snapshot_node.relation_name - == f'"{project.database}"."{project.test_schema}_alt"."snapshot_actual"' - ) - assert snapshot_node.meta == {"owner": "a_owner"} - - # write out schema.yml file and check again - write_file(models__schema_with_target_schema_yml, "models", "schema.yml") - manifest = run_dbt(["parse"]) - snapshot_node = manifest.nodes[snapshot_id] - assert snapshot_node.schema == "schema_from_schema_yml" - - -class CustomNamespace: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_pg_custom_namespaced__snapshot_sql} - - @pytest.fixture(scope="class") - def macros(self): - return { - "test_no_overlaps.sql": macros__test_no_overlaps_sql, - "custom.sql": macros_custom_snapshot__custom_sql, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -class TestBasicCustomNamespace(CustomNamespace): - def test_custom_namespace_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=1) - - -class TestRefCustomNamespace(CustomNamespace): - def test_custom_namespace_ref(self, project): - ref_setup(project, num_snapshot_models=1) - - -class CustomSnapshot: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_pg_custom__snapshot_sql} - - @pytest.fixture(scope="class") - def macros(self): - return { - "test_no_overlaps.sql": macros__test_no_overlaps_sql, - "custom.sql": macros_custom_snapshot__custom_sql, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -class TestBasicCustomSnapshot(CustomSnapshot): - def test_custom_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=1) - - -class TestRefCustomSnapshot(CustomSnapshot): - def test_custom_ref(self, project): - ref_setup(project, num_snapshot_models=1) - - -class CheckCols: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_check_col__snapshot_sql} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -class TestBasicCheckCols(CheckCols): - def test_basic_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=2) - - -class TestRefCheckCols(CheckCols): - def test_check_cols_ref(self, project): - ref_setup(project, num_snapshot_models=2) - - -class ConfiguredCheckCols: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_check_col_noconfig__snapshot_sql} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def project_config_update(self): - snapshot_config = { - "snapshots": { - "test": { - "target_schema": "{{ target.schema }}", - "unique_key": "id || '-' || first_name", - "strategy": "check", - "check_cols": ["email"], - } - } - } - return snapshot_config - - -class TestBasicConfiguredCheckCols(ConfiguredCheckCols): - def test_configured_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=2) - - -class TestRefConfiguredCheckCols(ConfiguredCheckCols): - def test_configured_ref(self, project): - ref_setup(project, num_snapshot_models=2) - - -class UpdatedAtCheckCols: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_check_col_noconfig__snapshot_sql} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def project_config_update(self): - snapshot_config = { - "snapshots": { - "test": { - "target_schema": "{{ target.schema }}", - "unique_key": "id || '-' || first_name", - "strategy": "check", - "check_cols": "all", - "updated_at": "updated_at", - } - } - } - return snapshot_config - - -class TestBasicUpdatedAtCheckCols(UpdatedAtCheckCols): - def test_updated_at_snapshot(self, project): - snapshot_setup(project, num_snapshot_models=2) - - snapshot_expected_relation = relation_from_name(project.adapter, "snapshot_expected") - revived_records = project.run_sql( - """ - select id, updated_at, dbt_valid_from from {} - """.format( - snapshot_expected_relation - ), - fetch="all", - ) - for result in revived_records: - # result is a tuple, the updated_at is second and dbt_valid_from is latest - assert isinstance(result[1], datetime) - assert isinstance(result[2], datetime) - assert result[1].replace(tzinfo=pytz.UTC) == result[2].replace(tzinfo=pytz.UTC) - - -class TestRefUpdatedAtCheckCols(UpdatedAtCheckCols): - def test_updated_at_ref(self, project): - ref_setup(project, num_snapshot_models=2) diff --git a/tests/functional/simple_snapshot/test_changing_check_cols_snapshot.py b/tests/functional/simple_snapshot/test_changing_check_cols_snapshot.py deleted file mode 100644 index d5333536..00000000 --- a/tests/functional/simple_snapshot/test_changing_check_cols_snapshot.py +++ /dev/null @@ -1,127 +0,0 @@ -from dbt.tests.util import check_relations_equal, run_dbt -import pytest - - -snapshot_sql = """ -{% snapshot snapshot_check_cols_new_column %} - {{ - config( - target_database=database, - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=var("check_cols", ['name']), - updated_at="'" ~ var("updated_at") ~ "'::timestamp", - ) - }} - - {% if var('version') == 1 %} - - select 1 as id, 'foo' as name - - {% else %} - - select 1 as id, 'foo' as name, 'bar' as other - - {% endif %} - -{% endsnapshot %} -""" - -expected_csv = """ -id,name,other,dbt_scd_id,dbt_updated_at,dbt_valid_from,dbt_valid_to -1,foo,NULL,0d73ad1b216ad884c9f7395d799c912c,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000,2016-07-02 00:00:00.000 -1,foo,bar,7df3783934a6a707d51254859260b9ff,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, -""".lstrip() - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot_check_cols_new_column.sql": snapshot_sql} - - -@pytest.fixture(scope="class") -def seeds(): - return {"snapshot_check_cols_new_column_expected.csv": expected_csv} - - -@pytest.fixture(scope="class") -def project_config_update(): - return { - "seeds": { - "quote_columns": False, - "test": { - "snapshot_check_cols_new_column_expected": { - "+column_types": { - "dbt_updated_at": "timestamp without time zone", - "dbt_valid_from": "timestamp without time zone", - "dbt_valid_to": "timestamp without time zone", - }, - }, - }, - }, - } - - -def run_check_cols_snapshot_with_schema_change(project, check_cols_override=None): - """ - Test that snapshots using the "check" strategy and explicit check_cols support adding columns. - - Approach: - 1. Take a snapshot that checks a single non-id column - 2. Add a new column to the data - 3. Take a snapshot that checks the new non-id column too - - As long as no error is thrown, then the snapshot was successful - """ - - check_cols = check_cols_override or ["name", "other"] - - # 1. Create a table that represents the expected data after a series of snapshots - vars_dict = {"version": 1, "updated_at": "2016-07-01"} - results = run_dbt(["seed", "--show", "--vars", str(vars_dict)]) - assert len(results) == 1 - - # Snapshot 1 - # Use only 'name' for check_cols - vars_dict = {"version": 1, "check_cols": [check_cols[0]], "updated_at": "2016-07-01"} - results = run_dbt(["snapshot", "--vars", str(vars_dict)]) - assert len(results) == 1 - - # Snapshot 2 - # Use both 'name' and 'other' for check_cols - vars_dict = {"version": 2, "check_cols": check_cols, "updated_at": "2016-07-02"} - results = run_dbt(["snapshot", "--vars", str(vars_dict)]) - assert len(results) == 1 - - check_relations_equal( - project.adapter, - ["snapshot_check_cols_new_column", "snapshot_check_cols_new_column_expected"], - compare_snapshot_cols=True, - ) - - # Snapshot 3 - # Run it again. Nothing has changed — ensure we don't detect changes - vars_dict = {"version": 2, "check_cols": check_cols, "updated_at": "2016-07-02"} - results = run_dbt(["snapshot", "--vars", str(vars_dict)]) - assert len(results) == 1 - - check_relations_equal( - project.adapter, - ["snapshot_check_cols_new_column", "snapshot_check_cols_new_column_expected"], - compare_snapshot_cols=True, - ) - - -def test_check_cols_snapshot_with_schema_change(project): - run_check_cols_snapshot_with_schema_change(project) - - -def test_check_cols_snapshot_with_schema_change_and_mismatched_casing(project): - """ - Test that this still works if the database-stored version of 'name' + 'other' - differs from the user-configured 'NAME' and 'OTHER' - """ - run_check_cols_snapshot_with_schema_change( - project=project, check_cols_override=["NAME", "OTHER"] - ) diff --git a/tests/functional/simple_snapshot/test_changing_strategy_snapshot.py b/tests/functional/simple_snapshot/test_changing_strategy_snapshot.py deleted file mode 100644 index 5540eee5..00000000 --- a/tests/functional/simple_snapshot/test_changing_strategy_snapshot.py +++ /dev/null @@ -1,128 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql - - -test_snapshots_changing_strategy__test_snapshot_sql = """ - -{# /* - Given the repro case for the snapshot build, we'd - expect to see both records have color='pink' - in their most recent rows. -*/ #} - -with expected as ( - - select 1 as id, 'pink' as color union all - select 2 as id, 'pink' as color - -), - -actual as ( - - select id, color - from {{ ref('my_snapshot') }} - where color = 'pink' - and dbt_valid_to is null - -) - -select * from expected -except -select * from actual - -union all - -select * from actual -except -select * from expected -""" - - -snapshots_changing_strategy__snapshot_sql = """ - -{# - REPRO: - 1. Run with check strategy - 2. Add a new ts column and run with check strategy - 3. Run with timestamp strategy on new ts column - - Expect: new entry is added for changed rows in (3) -#} - - -{% snapshot my_snapshot %} - - {#--------------- Configuration ------------ #} - - {{ config( - target_schema=schema, - unique_key='id' - ) }} - - {% if var('strategy') == 'timestamp' %} - {{ config(strategy='timestamp', updated_at='updated_at') }} - {% else %} - {{ config(strategy='check', check_cols=['color']) }} - {% endif %} - - {#--------------- Test setup ------------ #} - - {% if var('step') == 1 %} - - select 1 as id, 'blue' as color - union all - select 2 as id, 'red' as color - - {% elif var('step') == 2 %} - - -- change id=1 color from blue to green - -- id=2 is unchanged when using the check strategy - select 1 as id, 'green' as color, '2020-01-01'::date as updated_at - union all - select 2 as id, 'red' as color, '2020-01-01'::date as updated_at - - {% elif var('step') == 3 %} - - -- bump timestamp for both records. Expect that after this runs - -- using the timestamp strategy, both ids should have the color - -- 'pink' in the database. This should be in the future b/c we're - -- going to compare to the check timestamp, which will be _now_ - select 1 as id, 'pink' as color, (now() + interval '1 day')::date as updated_at - union all - select 2 as id, 'pink' as color, (now() + interval '1 day')::date as updated_at - - {% endif %} - -{% endsnapshot %} -""" - - -@pytest.fixture(scope="class") -def models(): - return {"gen.sql": models_slow__gen_sql} - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_changing_strategy__snapshot_sql} - - -@pytest.fixture(scope="class") -def tests(): - return {"test_snapshot.sql": test_snapshots_changing_strategy__test_snapshot_sql} - - -def test_changing_strategy(project): - results = run_dbt(["snapshot", "--vars", "{strategy: check, step: 1}"]) - assert len(results) == 1 - - results = run_dbt(["snapshot", "--vars", "{strategy: check, step: 2}"]) - assert len(results) == 1 - - results = run_dbt(["snapshot", "--vars", "{strategy: timestamp, step: 3}"]) - assert len(results) == 1 - - results = run_dbt(["test"]) - assert len(results) == 1 diff --git a/tests/functional/simple_snapshot/test_check_cols_snapshot.py b/tests/functional/simple_snapshot/test_check_cols_snapshot.py deleted file mode 100644 index 2b2673df..00000000 --- a/tests/functional/simple_snapshot/test_check_cols_snapshot.py +++ /dev/null @@ -1,113 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -snapshot_sql = """ -{% snapshot check_cols_cycle %} - - {{ - config( - target_database=database, - target_schema=schema, - unique_key='id', - strategy='check', - check_cols=['color'] - ) - }} - - {% if var('version') == 1 %} - - select 1 as id, 'red' as color union all - select 2 as id, 'green' as color - - {% elif var('version') == 2 %} - - select 1 as id, 'blue' as color union all - select 2 as id, 'green' as color - - {% elif var('version') == 3 %} - - select 1 as id, 'red' as color union all - select 2 as id, 'pink' as color - - {% else %} - {% do exceptions.raise_compiler_error("Got bad version: " ~ var('version')) %} - {% endif %} - -{% endsnapshot %} -""" - -snapshot_test_sql = """ -with query as ( - - -- check that the current value for id=1 is red - select case when ( - select count(*) - from {{ ref('check_cols_cycle') }} - where id = 1 and color = 'red' and dbt_valid_to is null - ) = 1 then 0 else 1 end as failures - - union all - - -- check that the previous 'red' value for id=1 is invalidated - select case when ( - select count(*) - from {{ ref('check_cols_cycle') }} - where id = 1 and color = 'red' and dbt_valid_to is not null - ) = 1 then 0 else 1 end as failures - - union all - - -- check that there's only one current record for id=2 - select case when ( - select count(*) - from {{ ref('check_cols_cycle') }} - where id = 2 and color = 'pink' and dbt_valid_to is null - ) = 1 then 0 else 1 end as failures - - union all - - -- check that the previous value for id=2 is represented - select case when ( - select count(*) - from {{ ref('check_cols_cycle') }} - where id = 2 and color = 'green' and dbt_valid_to is not null - ) = 1 then 0 else 1 end as failures - - union all - - -- check that there are 5 records total in the table - select case when ( - select count(*) - from {{ ref('check_cols_cycle') }} - ) = 5 then 0 else 1 end as failures - -) - -select * -from query -where failures = 1 -""" - - -@pytest.fixture(scope="class") -def snapshots(): - return {"my_snapshot.sql": snapshot_sql} - - -@pytest.fixture(scope="class") -def tests(): - return {"my_test.sql": snapshot_test_sql} - - -def test_simple_snapshot(project): - results = run_dbt(["snapshot", "--vars", "version: 1"]) - assert len(results) == 1 - - results = run_dbt(["snapshot", "--vars", "version: 2"]) - assert len(results) == 1 - - results = run_dbt(["snapshot", "--vars", "version: 3"]) - assert len(results) == 1 - - run_dbt(["test", "--select", "test_type:singular", "--vars", "version: 3"]) diff --git a/tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py b/tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py deleted file mode 100644 index 0c99d85e..00000000 --- a/tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py +++ /dev/null @@ -1,114 +0,0 @@ -from dbt.tests.util import check_relations_equal, run_dbt -import pytest - - -snapshot_sql = """ -{% snapshot snapshot_check_cols_updated_at_actual %} - {{ - config( - target_database=database, - target_schema=schema, - unique_key='id', - strategy='check', - check_cols='all', - updated_at="'" ~ var("updated_at") ~ "'::timestamp", - ) - }} - - {% if var('version') == 1 %} - - select 'a' as id, 10 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col union all - select 'b' as id, 20 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col - - {% elif var('version') == 2 %} - - select 'a' as id, 30 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col union all - select 'b' as id, 20 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col union all - select 'c' as id, 40 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col - - {% else %} - - select 'a' as id, 30 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col union all - select 'c' as id, 40 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col - - {% endif %} - -{% endsnapshot %} -""" - -expected_csv = """ -id,counter,timestamp_col,dbt_scd_id,dbt_updated_at,dbt_valid_from,dbt_valid_to -a,10,2016-01-01 00:00:00.000,927354aa091feffd9437ead0bdae7ae1,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000,2016-07-02 00:00:00.000 -b,20,2016-01-01 00:00:00.000,40ace4cbf8629f1720ec8a529ed76f8c,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000, -a,30,2016-01-02 00:00:00.000,e9133f2b302c50e36f43e770944cec9b,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, -c,40,2016-01-02 00:00:00.000,09d33d35101e788c152f65d0530b6837,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, -""".lstrip() - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot_check_cols_updated_at_actual.sql": snapshot_sql} - - -@pytest.fixture(scope="class") -def seeds(): - return {"snapshot_check_cols_updated_at_expected.csv": expected_csv} - - -@pytest.fixture(scope="class") -def project_config_update(): - return { - "seeds": { - "quote_columns": False, - "test": { - "snapshot_check_cols_updated_at_expected": { - "+column_types": { - "timestamp_col": "timestamp without time zone", - "dbt_updated_at": "timestamp without time zone", - "dbt_valid_from": "timestamp without time zone", - "dbt_valid_to": "timestamp without time zone", - }, - }, - }, - }, - } - - -def test_simple_snapshot(project): - """ - Test that the `dbt_updated_at` column reflects the `updated_at` timestamp expression in the config. - - Approach: - 1. Create a table that represents the expected data after a series of snapshots - - Use dbt seed to create the expected relation (`snapshot_check_cols_updated_at_expected`) - 2. Execute a series of snapshots to create the data - - Use a series of (3) dbt snapshot commands to create the actual relation (`snapshot_check_cols_updated_at_actual`) - - The logic can switch between 3 different versions of the data (depending on the `version` number) - - The `updated_at` value is passed in via `--vars` and cast to a timestamp in the snapshot config - 3. Compare the two relations for equality - """ - - # 1. Create a table that represents the expected data after a series of snapshots - results = run_dbt(["seed", "--show", "--vars", "{version: 1, updated_at: 2016-07-01}"]) - assert len(results) == 1 - - # 2. Execute a series of snapshots to create the data - - # Snapshot day 1 - results = run_dbt(["snapshot", "--vars", "{version: 1, updated_at: 2016-07-01}"]) - assert len(results) == 1 - - # Snapshot day 2 - results = run_dbt(["snapshot", "--vars", "{version: 2, updated_at: 2016-07-02}"]) - assert len(results) == 1 - - # Snapshot day 3 - results = run_dbt(["snapshot", "--vars", "{version: 3, updated_at: 2016-07-03}"]) - assert len(results) == 1 - - # 3. Compare the two relations for equality - check_relations_equal( - project.adapter, - ["snapshot_check_cols_updated_at_actual", "snapshot_check_cols_updated_at_expected"], - compare_snapshot_cols=True, - ) diff --git a/tests/functional/simple_snapshot/test_comment_ending_snapshot.py b/tests/functional/simple_snapshot/test_comment_ending_snapshot.py deleted file mode 100644 index ab21b641..00000000 --- a/tests/functional/simple_snapshot/test_comment_ending_snapshot.py +++ /dev/null @@ -1,36 +0,0 @@ -import os - -from dbt.tests.util import run_dbt -import pytest - - -snapshots_with_comment_at_end__snapshot_sql = """ -{% snapshot snapshot_actual %} - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id', - strategy='check', - check_cols=['email'], - ) - }} - select * from {{target.database}}.{{schema}}.seed - -- Test comment to prevent recurrence of https://github.com/dbt-labs/dbt-core/issues/6781 -{% endsnapshot %} -""" - - -class TestSnapshotsWithCommentAtEnd: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_with_comment_at_end__snapshot_sql} - - def test_comment_ending(self, project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - # N.B. Snapshot is run twice to ensure snapshot_check_all_get_existing_columns is fully run - # (it exits early if the table doesn't already exist) - run_dbt(["snapshot"]) - results = run_dbt(["snapshot"]) - assert len(results) == 1 diff --git a/tests/functional/simple_snapshot/test_cross_schema_snapshot.py b/tests/functional/simple_snapshot/test_cross_schema_snapshot.py deleted file mode 100644 index 1072a5aa..00000000 --- a/tests/functional/simple_snapshot/test_cross_schema_snapshot.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - models__ref_snapshot_sql, - models__schema_yml, - snapshots_pg__snapshot_sql, -) - - -NUM_SNAPSHOT_MODELS = 1 - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_pg__snapshot_sql} - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - -@pytest.fixture(scope="class") -def macros(): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - -def test_cross_schema_snapshot(project): - # populate seed and snapshot tables - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - - target_schema = "{}_snapshotted".format(project.test_schema) - - # create a snapshot using the new schema - results = run_dbt(["snapshot", "--vars", '{{"target_schema": "{}"}}'.format(target_schema)]) - assert len(results) == NUM_SNAPSHOT_MODELS - - # run dbt from test_schema with a ref to to new target_schema - results = run_dbt(["run", "--vars", '{{"target_schema": {}}}'.format(target_schema)]) - assert len(results) == 1 diff --git a/tests/functional/simple_snapshot/test_hard_delete_snapshot.py b/tests/functional/simple_snapshot/test_hard_delete_snapshot.py deleted file mode 100644 index ab25bbfa..00000000 --- a/tests/functional/simple_snapshot/test_hard_delete_snapshot.py +++ /dev/null @@ -1,192 +0,0 @@ -from datetime import datetime, timedelta -import os - -from dbt.tests.adapter.utils.test_current_timestamp import is_aware -from dbt.tests.util import run_dbt, check_relations_equal -import pytest -import pytz - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - models__ref_snapshot_sql, - models__schema_yml, - snapshots_pg__snapshot_sql, -) - - -# These tests uses the same seed data, containing 20 records of which we hard delete the last 10. -# These deleted records set the dbt_valid_to to time the snapshot was ran. - - -def convert_to_aware(d: datetime) -> datetime: - # There are two types of datetime objects in Python: naive and aware - # Assume any dbt snapshot timestamp that is naive is meant to represent UTC - if d is None: - return d - elif is_aware(d): - return d - else: - return d.replace(tzinfo=pytz.UTC) - - -def is_close_datetime( - dt1: datetime, dt2: datetime, atol: timedelta = timedelta(microseconds=1) -) -> bool: - # Similar to pytest.approx, math.isclose, and numpy.isclose - # Use an absolute tolerance to compare datetimes that may not be perfectly equal. - # Two None values will compare as equal. - if dt1 is None and dt2 is None: - return True - elif dt1 is not None and dt2 is not None: - return (dt1 > (dt2 - atol)) and (dt1 < (dt2 + atol)) - else: - return False - - -def datetime_snapshot(): - NUM_SNAPSHOT_MODELS = 1 - begin_snapshot_datetime = datetime.now(pytz.UTC) - results = run_dbt(["snapshot", "--vars", "{invalidate_hard_deletes: true}"]) - assert len(results) == NUM_SNAPSHOT_MODELS - - return begin_snapshot_datetime - - -@pytest.fixture(scope="class", autouse=True) -def setUp(project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_pg__snapshot_sql} - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - -@pytest.fixture(scope="class") -def macros(): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - -def test_snapshot_hard_delete(project): - # run the first snapshot - datetime_snapshot() - - check_relations_equal(project.adapter, ["snapshot_expected", "snapshot_actual"]) - - invalidated_snapshot_datetime = None - revived_snapshot_datetime = None - - # hard delete last 10 records - project.run_sql( - "delete from {}.{}.seed where id >= 10;".format(project.database, project.test_schema) - ) - - # snapshot and assert invalidated - invalidated_snapshot_datetime = datetime_snapshot() - - snapshotted = project.run_sql( - """ - select - id, - dbt_valid_to - from {}.{}.snapshot_actual - order by id - """.format( - project.database, project.test_schema - ), - fetch="all", - ) - - assert len(snapshotted) == 20 - for result in snapshotted[10:]: - # result is a tuple, the dbt_valid_to column is the latest - assert isinstance(result[-1], datetime) - dbt_valid_to = convert_to_aware(result[-1]) - - # Plenty of wiggle room if clocks aren't perfectly sync'd, etc - assert is_close_datetime( - dbt_valid_to, invalidated_snapshot_datetime, timedelta(minutes=1) - ), f"SQL timestamp {dbt_valid_to.isoformat()} is not close enough to Python UTC {invalidated_snapshot_datetime.isoformat()}" - - # revive records - # Timestamp must have microseconds for tests below to be meaningful - # Assume `updated_at` is TIMESTAMP WITHOUT TIME ZONE that implicitly represents UTC - revival_timestamp = datetime.now(pytz.UTC).strftime("%Y-%m-%d %H:%M:%S.%f") - project.run_sql( - """ - insert into {}.{}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values - (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '{}'), - (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '{}') - """.format( - project.database, project.test_schema, revival_timestamp, revival_timestamp - ) - ) - - # snapshot and assert records were revived - # Note: the revived_snapshot_datetime here is later than the revival_timestamp above - revived_snapshot_datetime = datetime_snapshot() - - # records which weren't revived (id != 10, 11) - # dbt_valid_to is not null - invalidated_records = project.run_sql( - """ - select - id, - dbt_valid_to - from {}.{}.snapshot_actual - where dbt_valid_to is not null - order by id - """.format( - project.database, project.test_schema - ), - fetch="all", - ) - - assert len(invalidated_records) == 11 - for result in invalidated_records: - # result is a tuple, the dbt_valid_to column is the latest - assert isinstance(result[1], datetime) - dbt_valid_to = convert_to_aware(result[1]) - - # Plenty of wiggle room if clocks aren't perfectly sync'd, etc - assert is_close_datetime( - dbt_valid_to, invalidated_snapshot_datetime, timedelta(minutes=1) - ), f"SQL timestamp {dbt_valid_to.isoformat()} is not close enough to Python UTC {invalidated_snapshot_datetime.isoformat()}" - - # records which were revived (id = 10, 11) - # dbt_valid_to is null - revived_records = project.run_sql( - """ - select - id, - dbt_valid_from, - dbt_valid_to - from {}.{}.snapshot_actual - where dbt_valid_to is null - and id IN (10, 11) - """.format( - project.database, project.test_schema - ), - fetch="all", - ) - - assert len(revived_records) == 2 - for result in revived_records: - # result is a tuple, the dbt_valid_from is second and dbt_valid_to is latest - # dbt_valid_from is the same as the 'updated_at' added in the revived_rows - # dbt_valid_to is null - assert isinstance(result[1], datetime) - dbt_valid_from = convert_to_aware(result[1]) - dbt_valid_to = result[2] - - assert dbt_valid_from <= revived_snapshot_datetime - assert dbt_valid_to is None diff --git a/tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py b/tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py deleted file mode 100644 index 1ee8fa40..00000000 --- a/tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py +++ /dev/null @@ -1,67 +0,0 @@ -import os - -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - macros_custom_snapshot__custom_sql, - models__ref_snapshot_sql, - models__schema_yml, - seeds__seed_csv, - seeds__seed_newcol_csv, -) - - -NUM_SNAPSHOT_MODELS = 1 - - -snapshots_pg_custom_invalid__snapshot_sql = """ -{% snapshot snapshot_actual %} - {# this custom strategy does not exist in the 'dbt' package #} - {{ - config( - target_database=var('target_database', database), - target_schema=var('target_schema', schema), - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='dbt.custom', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{target.schema}}.seed - -{% endsnapshot %} -""" - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshots.sql": snapshots_pg_custom_invalid__snapshot_sql} - - -@pytest.fixture(scope="class") -def macros(): - return { - "test_no_overlaps.sql": macros__test_no_overlaps_sql, - "custom.sql": macros_custom_snapshot__custom_sql, - } - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - -@pytest.fixture(scope="class") -def seeds(): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -def test_custom_snapshot_invalid_namespace(project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - results = run_dbt(["snapshot"], expect_pass=False) - assert len(results) == NUM_SNAPSHOT_MODELS diff --git a/tests/functional/simple_snapshot/test_long_text_snapshot.py b/tests/functional/simple_snapshot/test_long_text_snapshot.py deleted file mode 100644 index 0793a3fc..00000000 --- a/tests/functional/simple_snapshot/test_long_text_snapshot.py +++ /dev/null @@ -1,70 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - models__ref_snapshot_sql, - models__schema_yml, -) - - -seed_longtext_sql = """ -create table {database}.{schema}.super_long ( - id INTEGER, - longstring TEXT, - updated_at TIMESTAMP WITHOUT TIME ZONE -); - -insert into {database}.{schema}.super_long (id, longstring, updated_at) VALUES -(1, 'short', current_timestamp), -(2, repeat('a', 500), current_timestamp); -""" - -snapshots_longtext__snapshot_sql = """ -{% snapshot snapshot_actual %} - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id', - strategy='timestamp', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{schema}}.super_long -{% endsnapshot %} -""" - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_longtext__snapshot_sql} - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - -@pytest.fixture(scope="class") -def macros(): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - -def test_long_text(project): - project.run_sql(seed_longtext_sql) - - results = run_dbt(["snapshot"]) - assert len(results) == 1 - - with project.adapter.connection_named("test"): - status, results = project.adapter.execute( - "select * from {}.{}.snapshot_actual".format(project.database, project.test_schema), - fetch=True, - ) - assert len(results) == 2 - got_names = set(r.get("longstring") for r in results) - assert got_names == {"a" * 500, "short"} diff --git a/tests/functional/simple_snapshot/test_missing_strategy_snapshot.py b/tests/functional/simple_snapshot/test_missing_strategy_snapshot.py deleted file mode 100644 index 3a28bc77..00000000 --- a/tests/functional/simple_snapshot/test_missing_strategy_snapshot.py +++ /dev/null @@ -1,51 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - models__ref_snapshot_sql, - models__schema_yml, -) - - -snapshots_invalid__snapshot_sql = """ -{# make sure to never name this anything with `target_schema` in the name, or the test will be invalid! #} -{% snapshot missing_field_target_underscore_schema %} - {# missing the mandatory target_schema parameter #} - {{ - config( - unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', - updated_at='updated_at', - ) - }} - select * from {{target.database}}.{{schema}}.seed - -{% endsnapshot %} -""" - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_invalid__snapshot_sql} - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - -@pytest.fixture(scope="class") -def macros(): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - -def test_missing_strategy(project): - with pytest.raises(ParsingError) as exc: - run_dbt(["compile"], expect_pass=False) - - assert "Snapshots must be configured with a 'strategy'" in str(exc.value) diff --git a/tests/functional/simple_snapshot/test_renamed_source_snapshot.py b/tests/functional/simple_snapshot/test_renamed_source_snapshot.py deleted file mode 100644 index 23db614b..00000000 --- a/tests/functional/simple_snapshot/test_renamed_source_snapshot.py +++ /dev/null @@ -1,74 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - macros_custom_snapshot__custom_sql, - seeds__seed_csv, - seeds__seed_newcol_csv, -) - - -snapshots_checkall__snapshot_sql = """ -{% snapshot my_snapshot %} - {{ config(check_cols='all', unique_key='id', strategy='check', target_database=database, target_schema=schema) }} - select * from {{ ref(var('seed_name', 'seed')) }} -{% endsnapshot %} -""" - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_checkall__snapshot_sql} - - -@pytest.fixture(scope="class") -def macros(): - return { - "test_no_overlaps.sql": macros__test_no_overlaps_sql, - "custom.sql": macros_custom_snapshot__custom_sql, - } - - -@pytest.fixture(scope="class") -def seeds(): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - -def test_renamed_source(project): - run_dbt(["seed"]) - run_dbt(["snapshot"]) - database = project.database - results = project.run_sql( - "select * from {}.{}.my_snapshot".format(database, project.test_schema), - fetch="all", - ) - assert len(results) == 3 - for result in results: - assert len(result) == 6 - - # over ride the ref var in the snapshot definition to use a seed with an additional column, last_name - run_dbt(["snapshot", "--vars", "{seed_name: seed_newcol}"]) - results = project.run_sql( - "select * from {}.{}.my_snapshot where last_name is not NULL".format( - database, project.test_schema - ), - fetch="all", - ) - assert len(results) == 3 - - for result in results: - # new column - assert len(result) == 7 - assert result[-1] is not None - - results = project.run_sql( - "select * from {}.{}.my_snapshot where last_name is NULL".format( - database, project.test_schema - ), - fetch="all", - ) - assert len(results) == 3 - for result in results: - # new column - assert len(result) == 7 diff --git a/tests/functional/simple_snapshot/test_select_exclude_snapshot.py b/tests/functional/simple_snapshot/test_select_exclude_snapshot.py deleted file mode 100644 index ac2b4bc9..00000000 --- a/tests/functional/simple_snapshot/test_select_exclude_snapshot.py +++ /dev/null @@ -1,161 +0,0 @@ -import os - -from dbt.tests.util import ( - check_relations_equal, - check_table_does_not_exist, - run_dbt, -) -import pytest - -from tests.functional.simple_snapshot.fixtures import ( - macros__test_no_overlaps_sql, - models__ref_snapshot_sql, - models__schema_yml, - seeds__seed_csv, - seeds__seed_newcol_csv, - snapshots_pg__snapshot_sql, - snapshots_select__snapshot_sql, - snapshots_select_noconfig__snapshot_sql, -) - - -def all_snapshots(project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - - results = run_dbt(["snapshot"]) - assert len(results) == 4 - - check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) - check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) - check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) - check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) - - path = os.path.join(project.test_data_dir, "invalidate_postgres.sql") - project.run_sql_file(path) - - path = os.path.join(project.test_data_dir, "update.sql") - project.run_sql_file(path) - - results = run_dbt(["snapshot"]) - assert len(results) == 4 - check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) - check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) - check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) - check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) - - -def exclude_snapshots(project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - results = run_dbt(["snapshot", "--exclude", "snapshot_castillo"]) - assert len(results) == 3 - - check_table_does_not_exist(project.adapter, "snapshot_castillo") - check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) - check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) - check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) - - -def select_snapshots(project): - path = os.path.join(project.test_data_dir, "seed_pg.sql") - project.run_sql_file(path) - results = run_dbt(["snapshot", "--select", "snapshot_castillo"]) - assert len(results) == 1 - - check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) - check_table_does_not_exist(project.adapter, "snapshot_alvarez") - check_table_does_not_exist(project.adapter, "snapshot_kelly") - check_table_does_not_exist(project.adapter, "snapshot_actual") - - -# all of the tests below use one of both of the above tests with -# various combinations of snapshots and macros -class SelectBasicSetup: - @pytest.fixture(scope="class") - def snapshots(self): - return { - "snapshot.sql": snapshots_pg__snapshot_sql, - "snapshot_select.sql": snapshots_select__snapshot_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - -class TestAllBasic(SelectBasicSetup): - def test_all_snapshots(self, project): - all_snapshots(project) - - -class TestExcludeBasic(SelectBasicSetup): - def test_exclude_snapshots(self, project): - exclude_snapshots(project) - - -class TestSelectBasic(SelectBasicSetup): - def test_select_snapshots(self, project): - select_snapshots(project) - - -class SelectConfiguredSetup: - @pytest.fixture(scope="class") - def snapshots(self): - return {"snapshot.sql": snapshots_select_noconfig__snapshot_sql} - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": models__schema_yml, - "ref_snapshot.sql": models__ref_snapshot_sql, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} - - # TODO: don't have access to project here so this breaks - @pytest.fixture(scope="class") - def project_config_update(self): - snapshot_config = { - "snapshots": { - "test": { - "target_schema": "{{ target.schema }}", - "unique_key": "id || '-' || first_name", - "strategy": "timestamp", - "updated_at": "updated_at", - } - } - } - return snapshot_config - - -class TestConfigured(SelectConfiguredSetup): - def test_all_configured_snapshots(self, project): - all_snapshots(project) - - -class TestConfiguredExclude(SelectConfiguredSetup): - def test_exclude_configured_snapshots(self, project): - exclude_snapshots(project) - - -class TestConfiguredSelect(SelectConfiguredSetup): - def test_select_configured_snapshots(self, project): - select_snapshots(project) diff --git a/tests/functional/simple_snapshot/test_slow_query_snapshot.py b/tests/functional/simple_snapshot/test_slow_query_snapshot.py deleted file mode 100644 index a65b6cb3..00000000 --- a/tests/functional/simple_snapshot/test_slow_query_snapshot.py +++ /dev/null @@ -1,82 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql - - -snapshots_slow__snapshot_sql = """ - -{% snapshot my_slow_snapshot %} - - {{ - config( - target_database=var('target_database', database), - target_schema=schema, - unique_key='id', - strategy='timestamp', - updated_at='updated_at' - ) - }} - - select - id, - updated_at, - seconds - - from {{ ref('gen') }} - -{% endsnapshot %} -""" - - -test_snapshots_slow__test_timestamps_sql = """ - -/* - Assert that the dbt_valid_from of the latest record - is equal to the dbt_valid_to of the previous record -*/ - -with snapshot as ( - - select * from {{ ref('my_slow_snapshot') }} - -) - -select - snap1.id, - snap1.dbt_valid_from as new_valid_from, - snap2.dbt_valid_from as old_valid_from, - snap2.dbt_valid_to as old_valid_to - -from snapshot as snap1 -join snapshot as snap2 on snap1.id = snap2.id -where snap1.dbt_valid_to is null - and snap2.dbt_valid_to is not null - and snap1.dbt_valid_from != snap2.dbt_valid_to -""" - - -@pytest.fixture(scope="class") -def models(): - return {"gen.sql": models_slow__gen_sql} - - -@pytest.fixture(scope="class") -def snapshots(): - return {"snapshot.sql": snapshots_slow__snapshot_sql} - - -@pytest.fixture(scope="class") -def tests(): - return {"test_timestamps.sql": test_snapshots_slow__test_timestamps_sql} - - -def test_slow(project): - results = run_dbt(["snapshot"]) - assert len(results) == 1 - - results = run_dbt(["snapshot"]) - assert len(results) == 1 - - results = run_dbt(["test"]) - assert len(results) == 1 diff --git a/tests/functional/test_selection/conftest.py b/tests/functional/test_selection/conftest.py deleted file mode 100644 index 2faa9e34..00000000 --- a/tests/functional/test_selection/conftest.py +++ /dev/null @@ -1,96 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -import pytest - - -tests__cf_a_b_sql = """ -select * from {{ ref('model_a') }} -cross join {{ ref('model_b') }} -where false -""" - -tests__cf_a_src_sql = """ -select * from {{ ref('model_a') }} -cross join {{ source('my_src', 'my_tbl') }} -where false -""" - -tests__just_a_sql = """ -{{ config(tags = ['data_test_tag']) }} - -select * from {{ ref('model_a') }} -where false -""" - -models__schema_yml = """ -version: 2 - -sources: - - name: my_src - schema: "{{ target.schema }}" - tables: - - name: my_tbl - identifier: model_b - columns: - - name: fun - data_tests: - - unique - -models: - - name: model_a - columns: - - name: fun - tags: [column_level_tag] - data_tests: - - unique - - relationships: - to: ref('model_b') - field: fun - tags: [test_level_tag] - - relationships: - to: source('my_src', 'my_tbl') - field: fun -""" - -models__model_b_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select 1 as fun -""" - -models__model_a_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select * FROM {{ref('model_b')}} -""" - - -@pytest.fixture(scope="class") -def tests(): - return { - "cf_a_b.sql": tests__cf_a_b_sql, - "cf_a_src.sql": tests__cf_a_src_sql, - "just_a.sql": tests__just_a_sql, - } - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "model_b.sql": models__model_b_sql, - "model_a.sql": models__model_a_sql, - } - - -@pytest.fixture(scope="class") -def project_files( - project_root, - tests, - models, -): - write_project_files(project_root, "tests", tests) - write_project_files(project_root, "models", models) diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py deleted file mode 100644 index d17f27d7..00000000 --- a/tests/functional/test_selection/test_selection_expansion.py +++ /dev/null @@ -1,567 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -class TestSelectionExpansion: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"config-version": 2, "test-paths": ["tests"]} - - def list_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - list_args = ["ls", "--resource-type", "test"] - if include: - list_args.extend(("--select", include)) - if exclude: - list_args.extend(("--exclude", exclude)) - if indirect_selection: - list_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - list_args.extend(("--selector", selector_name)) - - listed = run_dbt(list_args) - assert len(listed) == len(expected_tests) - - test_names = [name.split(".")[-1] for name in listed] - assert sorted(test_names) == sorted(expected_tests) - - def run_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - results = run_dbt(["run"]) - assert len(results) == 2 - - test_args = ["test"] - if include: - test_args.extend(("--models", include)) - if exclude: - test_args.extend(("--exclude", exclude)) - if indirect_selection: - test_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - test_args.extend(("--selector", selector_name)) - - results = run_dbt(test_args) - tests_run = [r.node.name for r in results] - assert len(tests_run) == len(expected_tests) - - assert sorted(tests_run) == sorted(expected_tests) - - def test_all_tests_no_specifiers( - self, - project, - ): - select = None - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_alone( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_model_b( - self, - project, - ): - select = "model_a model_b" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_sources( - self, - project, - ): - select = "model_a source:*" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "source_unique_my_src_my_tbl_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_model_b( - self, - project, - ): - select = None - exclude = "model_b" - expected = [ - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test_cautious( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_exclude_specific_test_buildable( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "just_a", - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_only_generic( - self, - project, - ): - select = "test_type:generic" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_unset( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_eager( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_cautious( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert( - select, exclude, expected, indirect_selection=indirect_selection - ) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_only_singular( - self, - project, - ): - select = "test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_name_intersection( - self, - project, - ): - select = "model_a,test_name:unique" - exclude = None - expected = ["unique_model_a_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_tag_test_name_intersection( - self, - project, - ): - select = "tag:a_or_b,test_name:relationships" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_select_column_level_tag( - self, - project, - ): - select = "tag:column_level_tag" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_column_level_tag( - self, - project, - ): - select = None - exclude = "tag:column_level_tag" - expected = ["cf_a_b", "cf_a_src", "just_a", "source_unique_my_src_my_tbl_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_level_tag( - self, - project, - ): - select = "tag:test_level_tag" - exclude = None - expected = ["relationships_model_a_fun__fun__ref_model_b_"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_data_test_tag( - self, - project, - ): - select = "model_a" - exclude = "tag:data_test_tag" - expected = [ - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection_eager( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "eager" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_cautious( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "just_a", - "unique_model_a_fun", - ] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_buildable( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_exclude_unique_tests( - self, - project, - ): - select = "model_a" - exclude = "test_name:unique" - indirect_selection = "eager" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_model_a_indirect_selection_empty(self, project): - results = run_dbt(["ls", "--indirect-selection", "empty", "--select", "model_a"]) - assert len(results) == 1 - - -class TestExpansionWithSelectors(TestSelectionExpansion): - @pytest.fixture(scope="class") - def selectors(self): - return """ - selectors: - - name: model_a_unset_indirect_selection - definition: - method: fqn - value: model_a - - name: model_a_cautious_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "cautious" - - name: model_a_eager_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "eager" - - name: model_a_buildable_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "buildable" - """ - - def test_selector_model_a_unset_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - - def test_selector_model_a_cautious_indirect_selection( - self, - project, - ): - expected = ["just_a", "unique_model_a_fun"] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - - def test_selector_model_a_eager_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - - def test_selector_model_a_buildable_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - )