From c4f89cc45dabdc93c0fd63ca561017b43fd45a94 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 4 Jun 2024 15:00:02 -0500 Subject: [PATCH 01/10] init attempt of adding logger.info for model_hooks --- dbt/adapters/snowflake/impl.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index e3bc3ae0b..74573867e 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -7,6 +7,7 @@ from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.sql import SQLAdapter +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, @@ -19,6 +20,8 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +logger = AdapterLogger("Snowflake") + @dataclass class SnowflakeConfig(AdapterConfig): @@ -99,15 +102,20 @@ def _use_warehouse(self, warehouse: str): def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]: default_warehouse = self.config.credentials.warehouse warehouse = config.get("snowflake_warehouse", default_warehouse) + logger.info(f"Running pre_model_hook with config: {config}") + logger.info(f"Default warehouse: {default_warehouse}, Selected warehouse: {warehouse}") if warehouse == default_warehouse or warehouse is None: return None previous = self._get_warehouse() self._use_warehouse(warehouse) + logger.info(f"Changed warehouse from {previous} to {warehouse}") return previous def post_model_hook(self, config: Mapping[str, Any], context: Optional[str]) -> None: + logger.info(f"Running post_model_hook with config: {config} and context: {context}") if context is not None: self._use_warehouse(context) + logger.info(f"Restored warehouse to {context}") def list_schemas(self, database: str) -> List[str]: try: From 9d50d06ecee74318c9e338c472530a6a43f47cbc Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 4 Jun 2024 15:52:33 -0500 Subject: [PATCH 02/10] modify logging to not post full config (unnessecary on every run) --- dbt/adapters/snowflake/impl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 74573867e..2649cccfb 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -102,17 +102,17 @@ def _use_warehouse(self, warehouse: str): def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]: default_warehouse = self.config.credentials.warehouse warehouse = config.get("snowflake_warehouse", default_warehouse) - logger.info(f"Running pre_model_hook with config: {config}") + logger.info("Running pre_model_hook:") logger.info(f"Default warehouse: {default_warehouse}, Selected warehouse: {warehouse}") if warehouse == default_warehouse or warehouse is None: return None previous = self._get_warehouse() self._use_warehouse(warehouse) - logger.info(f"Changed warehouse from {previous} to {warehouse}") + logger.info(f"Changed wareho use from {previous} to {warehouse}") return previous def post_model_hook(self, config: Mapping[str, Any], context: Optional[str]) -> None: - logger.info(f"Running post_model_hook with config: {config} and context: {context}") + logger.info(f"Running post_model_hook with config, and context: {context}") if context is not None: self._use_warehouse(context) logger.info(f"Restored warehouse to {context}") From c45b8017395ae599f222a18f78625cff4a2b8922 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 5 Jun 2024 11:04:01 -0500 Subject: [PATCH 03/10] init form of test added --- dev-requirements.txt | 5 +-- .../warehouse_test/test_warehouses.py | 31 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0906cba2e..f06f084fb 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ -# install latest changes in dbt-core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +# install latest changes in dbt-core" +-e /Users/matthewmcknight/git/dbt-core/core +# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/tests/functional/warehouse_test/test_warehouses.py b/tests/functional/warehouse_test/test_warehouses.py index 717784185..7ef7a2fda 100644 --- a/tests/functional/warehouse_test/test_warehouses.py +++ b/tests/functional/warehouse_test/test_warehouses.py @@ -3,35 +3,34 @@ import os - models__override_warehouse_sql = """ {{ config(snowflake_warehouse=env_var('SNOWFLAKE_TEST_ALT_WAREHOUSE', 'DBT_TEST_ALT'), materialized='table') }} select current_warehouse() as warehouse - """ models__expected_warehouse_sql = """ {{ config(materialized='table') }} select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse - """ models__invalid_warehouse_sql = """ {{ config(snowflake_warehouse='DBT_TEST_DOES_NOT_EXIST') }} select current_warehouse() as warehouse - """ project_config_models__override_warehouse_sql = """ {{ config(materialized='table') }} select current_warehouse() as warehouse - """ project_config_models__expected_warehouse_sql = """ {{ config(materialized='table') }} select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse +""" +project_config_models__invalid_warehouse_sql = """ +{{ config(materialized='table') }} +select current_warehouse() as warehouse """ @@ -90,3 +89,25 @@ def test_snowflake_override_ok(self, project): ] ) check_relations_equal(project.adapter, ["OVERRIDE_WAREHOUSE", "EXPECTED_WAREHOUSE"]) + + +class TestInvalidConfigWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "invalid_warehouse.sql": project_config_models__invalid_warehouse_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "models": { + "test": { + "snowflake_warehouse": "DBT_TEST_DOES_NOT_EXIST", + }, + }, + } + + def test_snowflake_override_invalid(self, project): + run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) From b7c647e6c578e30904cc940d70dfc85a3758c66c Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 5 Jun 2024 11:07:19 -0500 Subject: [PATCH 04/10] change core pointer in dev-requirements --- dev-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f06f084fb..358d359a3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,5 @@ # install latest changes in dbt-core" --e /Users/matthewmcknight/git/dbt-core/core -# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@mcknight/10198#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git From 0185022357669504ea0e352e0becf7c4886731e9 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 6 Jun 2024 10:57:54 -0500 Subject: [PATCH 05/10] removing logging as unneeded per feedback, and adding assert to new test --- dbt/adapters/snowflake/impl.py | 8 -------- tests/functional/warehouse_test/test_warehouses.py | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 2649cccfb..e3bc3ae0b 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -7,7 +7,6 @@ from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.sql import SQLAdapter -from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, @@ -20,8 +19,6 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values -logger = AdapterLogger("Snowflake") - @dataclass class SnowflakeConfig(AdapterConfig): @@ -102,20 +99,15 @@ def _use_warehouse(self, warehouse: str): def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]: default_warehouse = self.config.credentials.warehouse warehouse = config.get("snowflake_warehouse", default_warehouse) - logger.info("Running pre_model_hook:") - logger.info(f"Default warehouse: {default_warehouse}, Selected warehouse: {warehouse}") if warehouse == default_warehouse or warehouse is None: return None previous = self._get_warehouse() self._use_warehouse(warehouse) - logger.info(f"Changed wareho use from {previous} to {warehouse}") return previous def post_model_hook(self, config: Mapping[str, Any], context: Optional[str]) -> None: - logger.info(f"Running post_model_hook with config, and context: {context}") if context is not None: self._use_warehouse(context) - logger.info(f"Restored warehouse to {context}") def list_schemas(self, database: str) -> List[str]: try: diff --git a/tests/functional/warehouse_test/test_warehouses.py b/tests/functional/warehouse_test/test_warehouses.py index 7ef7a2fda..687ca7b75 100644 --- a/tests/functional/warehouse_test/test_warehouses.py +++ b/tests/functional/warehouse_test/test_warehouses.py @@ -110,4 +110,5 @@ def project_config_update(self): } def test_snowflake_override_invalid(self, project): - run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) + result = run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) + assert "Object does not exist, or operation cannot be performed" in result[0].message From ae16ab767ec27462c3132f5c9df1d35c604338ed Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 10 Jun 2024 14:43:39 -0500 Subject: [PATCH 06/10] add in passing test --- .../warehouse_test/test_warehouses.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/functional/warehouse_test/test_warehouses.py b/tests/functional/warehouse_test/test_warehouses.py index 687ca7b75..189eb313a 100644 --- a/tests/functional/warehouse_test/test_warehouses.py +++ b/tests/functional/warehouse_test/test_warehouses.py @@ -33,6 +33,11 @@ select current_warehouse() as warehouse """ +project_config_models__valid_warehouse_sql = """ +{{ config(materialized='table') }} +select current_warehouse() as warehouse +""" + class TestModelWarehouse: @pytest.fixture(scope="class") @@ -103,12 +108,32 @@ def project_config_update(self): return { "config-version": 2, "models": { - "test": { - "snowflake_warehouse": "DBT_TEST_DOES_NOT_EXIST", - }, + "test": {"snowflake_warehouse": "DBT_TEST_DOES_NOT_EXIST"}, }, } def test_snowflake_override_invalid(self, project): result = run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False) assert "Object does not exist, or operation cannot be performed" in result[0].message + + +class TestValidConfigWarehouse: + @pytest.fixture(scope="class") + def models(self): + return { + "valid_warehouse.sql": project_config_models__valid_warehouse_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "models": { + "test": { + "snowflake_warehouse": "DBT_TESTING", + }, + }, + } + + def test_snowflake_override_invalid(self, project): + result = run_dbt(["run", "--models", "valid_warehouse"]) From 305bf77ce8d992f851f94ec26f7e6f7324947463 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 10 Jun 2024 14:52:17 -0500 Subject: [PATCH 07/10] add in passing test --- tests/functional/warehouse_test/test_warehouses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/warehouse_test/test_warehouses.py b/tests/functional/warehouse_test/test_warehouses.py index 189eb313a..2b75f91f1 100644 --- a/tests/functional/warehouse_test/test_warehouses.py +++ b/tests/functional/warehouse_test/test_warehouses.py @@ -135,5 +135,6 @@ def project_config_update(self): }, } - def test_snowflake_override_invalid(self, project): + def test_snowflake_warehouse_valid(self, project): result = run_dbt(["run", "--models", "valid_warehouse"]) + assert "DBT_TESTING" in result[0].node.config.get("snowflake_warehouse") From 9533012e39d27bd16be18460eee0da3f12476eee Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 12 Jun 2024 14:57:04 -0500 Subject: [PATCH 08/10] consolidate sql fixtures --- tests/functional/warehouse_test/test_warehouses.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/functional/warehouse_test/test_warehouses.py b/tests/functional/warehouse_test/test_warehouses.py index 2b75f91f1..268473729 100644 --- a/tests/functional/warehouse_test/test_warehouses.py +++ b/tests/functional/warehouse_test/test_warehouses.py @@ -28,12 +28,7 @@ select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse """ -project_config_models__invalid_warehouse_sql = """ -{{ config(materialized='table') }} -select current_warehouse() as warehouse -""" - -project_config_models__valid_warehouse_sql = """ +project_config_models__warehouse_sql = """ {{ config(materialized='table') }} select current_warehouse() as warehouse """ @@ -100,7 +95,7 @@ class TestInvalidConfigWarehouse: @pytest.fixture(scope="class") def models(self): return { - "invalid_warehouse.sql": project_config_models__invalid_warehouse_sql, + "invalid_warehouse.sql": project_config_models__warehouse_sql, } @pytest.fixture(scope="class") @@ -121,7 +116,7 @@ class TestValidConfigWarehouse: @pytest.fixture(scope="class") def models(self): return { - "valid_warehouse.sql": project_config_models__valid_warehouse_sql, + "valid_warehouse.sql": project_config_models__warehouse_sql, } @pytest.fixture(scope="class") From 3f3714882dfa597099a85437defb4f10bf9bd375 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 13 Jun 2024 10:51:23 -0500 Subject: [PATCH 09/10] revert core pointer --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 358d359a3..8831de323 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # install latest changes in dbt-core" -git+https://github.com/dbt-labs/dbt-core.git@mcknight/10198#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git From e6b9ae4cd5d8c7f8754c8a7bf39a07e09af7de11 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Fri, 14 Jun 2024 14:16:11 -0500 Subject: [PATCH 10/10] remove unneeded quote --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8831de323..0906cba2e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -# install latest changes in dbt-core" +# install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter