From 809704c152c546a3c224e3780ddb35980f11347f Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 27 Feb 2024 15:37:52 -0500 Subject: [PATCH 1/4] Bump dbt-common<2.0 (#25) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6b58ab14..6b67b91a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "dbt-adapters>=0.1.0a6,<0.2.0", "psycopg2>=2.9,<3.0", # installed via dbt-adapters but used directly - "dbt-common<1.0", + "dbt-common<2.0", "agate>=1.0,<2.0", ] [project.urls] From 71102f8a0d5539206e8f4938a45659ce0e0eda0f Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:32:49 -0500 Subject: [PATCH 2/4] Remove saved query tests, they belong in dbt-core (#26) This PR only removed test cases. Failed tests are resolved via https://github.com/dbt-labs/dbt-postgres/pull/21. --- tests/functional/saved_queries/fixtures.py | 93 --------- .../saved_queries/test_saved_query_build.py | 41 ---- .../saved_queries/test_saved_query_configs.py | 186 ------------------ .../saved_queries/test_saved_query_parsing.py | 113 ----------- 4 files changed, 433 deletions(-) delete mode 100644 tests/functional/saved_queries/fixtures.py delete mode 100644 tests/functional/saved_queries/test_saved_query_build.py delete mode 100644 tests/functional/saved_queries/test_saved_query_configs.py delete mode 100644 tests/functional/saved_queries/test_saved_query_parsing.py diff --git a/tests/functional/saved_queries/fixtures.py b/tests/functional/saved_queries/fixtures.py deleted file mode 100644 index 68565d82..00000000 --- a/tests/functional/saved_queries/fixtures.py +++ /dev/null @@ -1,93 +0,0 @@ -saved_query_description = """ -{% docs saved_query_description %} My SavedQuery Description {% enddocs %} -""" - -saved_queries_yml = """ -version: 2 - -saved_queries: - - name: test_saved_query - description: "{{ doc('saved_query_description') }}" - label: Test Saved Query - query_params: - metrics: - - simple_metric - group_by: - - "Dimension('user__ds')" - where: - - "{{ Dimension('user__ds', 'DAY') }} <= now()" - - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" - exports: - - name: my_export - config: - alias: my_export_alias - export_as: table - schema: my_export_schema_name -""" - -saved_query_with_extra_config_attributes_yml = """ -version: 2 - -saved_queries: - - name: test_saved_query - description: "{{ doc('saved_query_description') }}" - label: Test Saved Query - query_params: - metrics: - - simple_metric - group_by: - - "Dimension('user__ds')" - where: - - "{{ Dimension('user__ds', 'DAY') }} <= now()" - - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" - exports: - - name: my_export - config: - my_random_config: 'I have this for some reason' - export_as: table -""" - -saved_query_with_export_configs_defined_at_saved_query_level_yml = """ -version: 2 - -saved_queries: - - name: test_saved_query - description: "{{ doc('saved_query_description') }}" - label: Test Saved Query - config: - export_as: table - schema: my_default_export_schema - query_params: - metrics: - - simple_metric - group_by: - - "Dimension('user__ds')" - where: - - "{{ Dimension('user__ds', 'DAY') }} <= now()" - - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" - exports: - - name: my_export - config: - export_as: view - schema: my_custom_export_schema - - name: my_export2 -""" - -saved_query_without_export_configs_defined_yml = """ -version: 2 - -saved_queries: - - name: test_saved_query - description: "{{ doc('saved_query_description') }}" - label: Test Saved Query - query_params: - metrics: - - simple_metric - group_by: - - "Dimension('user__ds')" - where: - - "{{ Dimension('user__ds', 'DAY') }} <= now()" - - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" - exports: - - name: my_export -""" diff --git a/tests/functional/saved_queries/test_saved_query_build.py b/tests/functional/saved_queries/test_saved_query_build.py deleted file mode 100644 index 19787e71..00000000 --- a/tests/functional/saved_queries/test_saved_query_build.py +++ /dev/null @@ -1,41 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - -from tests.functional.saved_queries.fixtures import ( - saved_queries_yml, - saved_query_description, -) -from tests.functional.semantic_models.fixtures import ( - fct_revenue_sql, - metricflow_time_spine_sql, - schema_yml, -) - - -class TestSavedQueryBuildNoOp: - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_queries_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - @pytest.fixture(scope="class") - def packages(self): - return """ -packages: - - package: dbt-labs/dbt_utils - version: 1.1.1 -""" - - def test_semantic_model_parsing(self, project): - run_dbt(["deps"]) - result = run_dbt(["build"]) - assert len(result.results) == 2 - assert "test_saved_query" not in [r.node.name for r in result.results] - result = run_dbt(["build", "--include-saved-query"]) - assert len(result.results) == 3 - assert "test_saved_query" in [r.node.name for r in result.results] diff --git a/tests/functional/saved_queries/test_saved_query_configs.py b/tests/functional/saved_queries/test_saved_query_configs.py deleted file mode 100644 index 685c54c2..00000000 --- a/tests/functional/saved_queries/test_saved_query_configs.py +++ /dev/null @@ -1,186 +0,0 @@ -from dbt.contracts.graph.manifest import Manifest -from dbt.tests.util import update_config_file -from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType -import pytest - -from tests.functional.configs.fixtures import BaseConfigProject -from tests.functional.dbt_runner import dbtTestRunner -from tests.functional.saved_queries.fixtures import ( - saved_queries_yml, - saved_query_description, - saved_query_with_export_configs_defined_at_saved_query_level_yml, - saved_query_with_extra_config_attributes_yml, - saved_query_without_export_configs_defined_yml, -) -from tests.functional.semantic_models.fixtures import ( - fct_revenue_sql, - metricflow_time_spine_sql, - schema_yml, -) - - -class TestSavedQueryConfigs(BaseConfigProject): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "saved-queries": { - "test": { - "test_saved_query": { - "+enabled": True, - "+export_as": ExportDestinationType.VIEW.value, - "+schema": "my_default_export_schema", - } - }, - }, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_query_with_extra_config_attributes_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_basic_saved_query_config( - self, - project, - ): - runner = dbtTestRunner() - - # parse with default fixture project config - result = runner.invoke(["parse"]) - assert result.success - assert isinstance(result.result, Manifest) - assert len(result.result.saved_queries) == 1 - saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] - assert saved_query.config.export_as == ExportDestinationType.VIEW - assert saved_query.config.schema == "my_default_export_schema" - - # disable the saved_query via project config and rerun - config_patch = {"saved-queries": {"test": {"test_saved_query": {"+enabled": False}}}} - update_config_file(config_patch, project.project_root, "dbt_project.yml") - result = runner.invoke(["parse"]) - assert result.success - assert len(result.result.saved_queries) == 0 - - -class TestExportConfigsWithAdditionalProperties(BaseConfigProject): - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_queries_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_extra_config_properties_dont_break_parsing(self, project): - runner = dbtTestRunner() - - # parse with default fixture project config - result = runner.invoke(["parse"]) - assert result.success - assert isinstance(result.result, Manifest) - assert len(result.result.saved_queries) == 1 - saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] - assert len(saved_query.exports) == 1 - assert saved_query.exports[0].config.__dict__.get("my_random_config") is None - - -class TestInheritingExportConfigFromSavedQueryConfig(BaseConfigProject): - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_query_with_export_configs_defined_at_saved_query_level_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_export_config_inherits_from_saved_query(self, project): - runner = dbtTestRunner() - - # parse with default fixture project config - result = runner.invoke(["parse"]) - assert result.success - assert isinstance(result.result, Manifest) - assert len(result.result.saved_queries) == 1 - saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] - assert len(saved_query.exports) == 2 - - # assert Export `my_export` has its configs defined from itself because they should take priority - export1 = next( - (export for export in saved_query.exports if export.name == "my_export"), None - ) - assert export1 is not None - assert export1.config.export_as == ExportDestinationType.VIEW - assert export1.config.export_as != saved_query.config.export_as - assert export1.config.schema_name == "my_custom_export_schema" - assert export1.config.schema_name != saved_query.config.schema - - # assert Export `my_export` has its configs defined from the saved_query because they should take priority - export2 = next( - (export for export in saved_query.exports if export.name == "my_export2"), None - ) - assert export2 is not None - assert export2.config.export_as == ExportDestinationType.TABLE - assert export2.config.export_as == saved_query.config.export_as - assert export2.config.schema_name == "my_default_export_schema" - assert export2.config.schema_name == saved_query.config.schema - - -class TestInheritingExportConfigsFromProject(BaseConfigProject): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "saved-queries": { - "test": { - "test_saved_query": { - "+export_as": ExportDestinationType.VIEW.value, - } - }, - }, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_query_without_export_configs_defined_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_export_config_inherits_from_project( - self, - project, - ): - runner = dbtTestRunner() - - # parse with default fixture project config - result = runner.invoke(["parse"]) - assert result.success - assert isinstance(result.result, Manifest) - assert len(result.result.saved_queries) == 1 - saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] - assert saved_query.config.export_as == ExportDestinationType.VIEW - - # change export's `export_as` to `TABLE` via project config - config_patch = { - "saved-queries": { - "test": {"test_saved_query": {"+export_as": ExportDestinationType.TABLE.value}} - } - } - update_config_file(config_patch, project.project_root, "dbt_project.yml") - result = runner.invoke(["parse"]) - assert result.success - assert isinstance(result.result, Manifest) - assert len(result.result.saved_queries) == 1 - saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] - assert saved_query.config.export_as == ExportDestinationType.TABLE diff --git a/tests/functional/saved_queries/test_saved_query_parsing.py b/tests/functional/saved_queries/test_saved_query_parsing.py deleted file mode 100644 index 73f63f1e..00000000 --- a/tests/functional/saved_queries/test_saved_query_parsing.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import List - -from dbt.contracts.graph.manifest import Manifest -from dbt.tests.util import write_file -from dbt_common.events.base_types import BaseEvent -from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType -import pytest - -from tests.functional.dbt_runner import dbtTestRunner -from tests.functional.saved_queries.fixtures import ( - saved_queries_yml, - saved_query_description, -) -from tests.functional.semantic_models.fixtures import ( - fct_revenue_sql, - metricflow_time_spine_sql, - schema_yml, -) - - -class TestSavedQueryParsing: - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_queries_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_semantic_model_parsing(self, project): - runner = dbtTestRunner() - result = runner.invoke(["parse", "--no-partial-parse"]) - assert result.success - assert isinstance(result.result, Manifest) - manifest = result.result - assert len(manifest.saved_queries) == 1 - saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] - assert saved_query.name == "test_saved_query" - assert len(saved_query.query_params.metrics) == 1 - assert len(saved_query.query_params.group_by) == 1 - assert len(saved_query.query_params.where.where_filters) == 2 - assert len(saved_query.depends_on.nodes) == 1 - assert saved_query.description == "My SavedQuery Description" - assert len(saved_query.exports) == 1 - assert saved_query.exports[0].name == "my_export" - assert saved_query.exports[0].config.alias == "my_export_alias" - assert saved_query.exports[0].config.export_as == ExportDestinationType.TABLE - assert saved_query.exports[0].config.schema_name == "my_export_schema_name" - - def test_saved_query_error(self, project): - error_schema_yml = saved_queries_yml.replace("simple_metric", "metric_not_found") - write_file(error_schema_yml, project.project_root, "models", "saved_queries.yml") - events: List[BaseEvent] = [] - runner = dbtTestRunner(callbacks=[events.append]) - - result = runner.invoke(["parse", "--no-partial-parse"]) - assert not result.success - validation_errors = [e for e in events if e.info.name == "MainEncounteredError"] - assert validation_errors - - -class TestSavedQueryPartialParsing: - @pytest.fixture(scope="class") - def models(self): - return { - "saved_queries.yml": saved_queries_yml, - "schema.yml": schema_yml, - "fct_revenue.sql": fct_revenue_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "docs.md": saved_query_description, - } - - def test_saved_query_metrics_changed(self, project): - # First, use the default saved_queries.yml to define our saved_queries, and - # run the dbt parse command - runner = dbtTestRunner() - result = runner.invoke(["parse"]) - assert result.success - - # Next, modify the default saved_queries.yml to change a detail of the saved - # query. - modified_saved_queries_yml = saved_queries_yml.replace("simple_metric", "txn_revenue") - write_file(modified_saved_queries_yml, project.project_root, "models", "saved_queries.yml") - - # Now, run the dbt parse command again. - result = runner.invoke(["parse"]) - assert result.success - - # Finally, verify that the manifest reflects the partially parsed change - manifest = result.result - saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] - assert len(saved_query.metrics) == 1 - assert saved_query.metrics[0] == "txn_revenue" - - def test_saved_query_deleted_partial_parsing(self, project): - # First, use the default saved_queries.yml to define our saved_query, and - # run the dbt parse command - runner = dbtTestRunner() - result = runner.invoke(["parse"]) - assert result.success - assert "saved_query.test.test_saved_query" in result.result.saved_queries - - # Next, modify the default saved_queries.yml to remove the saved query. - write_file("", project.project_root, "models", "saved_queries.yml") - - # Now, run the dbt parse command again. - result = runner.invoke(["parse"]) - assert result.success - - # Finally, verify that the manifest reflects the deletion - assert "saved_query.test.test_saved_query" not in result.result.saved_queries From dd4d8dade6908fc2ddf21eb39b8abbe4693d15ad Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:33:01 -0500 Subject: [PATCH 3/4] Update dependencies for 1.0.0b1 (#27) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6b67b91a..e7e283c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "dbt-adapters>=0.1.0a6,<0.2.0", + "dbt-adapters>=0.1.0a1,<2.0", "psycopg2>=2.9,<3.0", # installed via dbt-adapters but used directly "dbt-common<2.0", From c2e7104a6405f166b9ceb871ca7edbcbf3602824 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:10:23 -0500 Subject: [PATCH 4/4] Fix testing migration (#21) --- pyproject.toml | 2 +- .../functional/shared_tests/test_column_types.py | 4 ++-- .../shared_tests/test_hooks/data/seed_model.sql | 16 ++++++++++++++++ .../shared_tests/test_hooks/data/seed_run.sql | 16 ++++++++++++++++ .../shared_tests/{ => test_hooks}/test_hooks.py | 4 ++++ .../{ => test_simple_seed}/seed_bom.csv | 0 .../{ => test_simple_seed}/test_simple_seed.py | 4 ++++ 7 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/functional/shared_tests/test_hooks/data/seed_model.sql create mode 100644 tests/functional/shared_tests/test_hooks/data/seed_run.sql rename tests/functional/shared_tests/{ => test_hooks}/test_hooks.py (92%) rename tests/functional/shared_tests/{ => test_simple_seed}/seed_bom.csv (100%) rename tests/functional/shared_tests/{ => test_simple_seed}/test_simple_seed.py (89%) diff --git a/pyproject.toml b/pyproject.toml index e7e283c2..acef4e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "dbt-adapters>=0.1.0a1,<2.0", "psycopg2>=2.9,<3.0", # installed via dbt-adapters but used directly - "dbt-common<2.0", + "dbt-common>=0.1.0a1,<2.0", "agate>=1.0,<2.0", ] [project.urls] diff --git a/tests/functional/shared_tests/test_column_types.py b/tests/functional/shared_tests/test_column_types.py index f5037860..47dec245 100644 --- a/tests/functional/shared_tests/test_column_types.py +++ b/tests/functional/shared_tests/test_column_types.py @@ -1,5 +1,5 @@ -from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes +from dbt.tests.adapter.column_types.test_column_types import BasePostgresColumnTypes -class TestPostgresColumnTypes(BaseColumnTypes): +class TestPostgresColumnTypes(BasePostgresColumnTypes): pass diff --git a/tests/functional/shared_tests/test_hooks/data/seed_model.sql b/tests/functional/shared_tests/test_hooks/data/seed_model.sql new file mode 100644 index 00000000..6727acb3 --- /dev/null +++ b/tests/functional/shared_tests/test_hooks/data/seed_model.sql @@ -0,0 +1,16 @@ +drop table if exists {schema}.on_model_hook; + +create table {schema}.on_model_hook ( + test_state TEXT, -- start|end + target_dbname TEXT, + target_host TEXT, + target_name TEXT, + target_schema TEXT, + target_type TEXT, + target_user TEXT, + target_pass TEXT, + target_threads INTEGER, + run_started_at TEXT, + invocation_id TEXT, + thread_id TEXT +); diff --git a/tests/functional/shared_tests/test_hooks/data/seed_run.sql b/tests/functional/shared_tests/test_hooks/data/seed_run.sql new file mode 100644 index 00000000..079ed34a --- /dev/null +++ b/tests/functional/shared_tests/test_hooks/data/seed_run.sql @@ -0,0 +1,16 @@ +drop table if exists {schema}.on_run_hook; + +create table {schema}.on_run_hook ( + test_state TEXT, -- start|end + target_dbname TEXT, + target_host TEXT, + target_name TEXT, + target_schema TEXT, + target_type TEXT, + target_user TEXT, + target_pass TEXT, + target_threads INTEGER, + run_started_at TEXT, + invocation_id TEXT, + thread_id TEXT +); diff --git a/tests/functional/shared_tests/test_hooks.py b/tests/functional/shared_tests/test_hooks/test_hooks.py similarity index 92% rename from tests/functional/shared_tests/test_hooks.py rename to tests/functional/shared_tests/test_hooks/test_hooks.py index 3db138eb..84381130 100644 --- a/tests/functional/shared_tests/test_hooks.py +++ b/tests/functional/shared_tests/test_hooks/test_hooks.py @@ -1,3 +1,7 @@ +""" +This file needs to be in its own directory because it uses a `data` directory. +Placing this file in its own directory avoids collisions. +""" from dbt.tests.adapter.hooks.test_model_hooks import ( BasePrePostModelHooks, BaseHookRefs, diff --git a/tests/functional/shared_tests/seed_bom.csv b/tests/functional/shared_tests/test_simple_seed/seed_bom.csv similarity index 100% rename from tests/functional/shared_tests/seed_bom.csv rename to tests/functional/shared_tests/test_simple_seed/seed_bom.csv diff --git a/tests/functional/shared_tests/test_simple_seed.py b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py similarity index 89% rename from tests/functional/shared_tests/test_simple_seed.py rename to tests/functional/shared_tests/test_simple_seed/test_simple_seed.py index 97b8870a..cd849788 100644 --- a/tests/functional/shared_tests/test_simple_seed.py +++ b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py @@ -1,3 +1,7 @@ +""" +This file needs to be in its own directory because it creates a `data` directory at run time. +Placing this file in its own directory avoids collisions. +""" from dbt.tests.adapter.simple_seed.test_seed import ( BaseBasicSeedTests, BaseSeedConfigFullRefreshOn,