From e8a061171151a60b9eb446807194354244f8632f Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Mon, 9 Oct 2023 14:09:42 -0700 Subject: [PATCH] get it to work --- dbt/adapters/databricks/connections.py | 3 +- .../relation_configs/materialized_view.py | 49 ++++++++-------- .../test_materialized_views.py | 16 +++++- .../test_materialized_view.py | 57 ------------------- 4 files changed, 41 insertions(+), 84 deletions(-) delete mode 100644 tests/unit/relation_configs/test_materialized_view.py diff --git a/dbt/adapters/databricks/connections.py b/dbt/adapters/databricks/connections.py index 3889c64f5..0885a462b 100644 --- a/dbt/adapters/databricks/connections.py +++ b/dbt/adapters/databricks/connections.py @@ -100,7 +100,8 @@ def emit(self, record: logging.LogRecord) -> None: @dataclass class DatabricksCredentials(Credentials): - database: Optional[str] # type: ignore[assignment] + database: Optional[str] = None + schema: Optional[str] = None host: Optional[str] = None http_path: Optional[str] = None token: Optional[str] = None diff --git a/dbt/adapters/databricks/relation_configs/materialized_view.py b/dbt/adapters/databricks/relation_configs/materialized_view.py index 538a33469..e004e0259 100644 --- a/dbt/adapters/databricks/relation_configs/materialized_view.py +++ b/dbt/adapters/databricks/relation_configs/materialized_view.py @@ -45,9 +45,8 @@ class DatabricksMaterializedViewConfig(DatabricksRelationConfigBase, RelationCon schema_name: str database_name: str query: str - backup: bool = True partition: Optional[str] = None # to be done - schedule: Optional[str] = None + schedule: Optional[str] = None # to be done @property def path(self) -> str: @@ -84,17 +83,18 @@ def parse_model_node(cls, model_node: ModelNode) -> dict: "database_name": model_node.database, } - autorefresh_value = model_node.config.extra.get("schedule") - if autorefresh_value is not None: - config_dict["schedule"] = evaluate_bool(autorefresh_value) - if query := model_node.compiled_code: config_dict.update({"query": query.strip()}) - if model_node.config.get("partition"): - config_dict.update( - {"partition": DatabricksPartitionedByConfig.parse_model_node(model_node)} - ) + # TODO + # schedule = model_node.config.extra.get("schedule") + # if schedule is not None: + # config_dict["schedule"] = + + # if model_node.config.get("partition"): + # config_dict.update( + # {"partition": DatabricksPartitionedByConfig.parse_model_node(model_node)} + # ) return config_dict @@ -103,6 +103,7 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict: """ Translate agate objects from the database into a standard dictionary. + # TODO: Fix this, the description comes from Redshift Args: relation_results: the description of the materialized view from the database in this format: @@ -133,7 +134,7 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict: "mv_name": materialized_view.get("table"), "schema_name": materialized_view.get("schema"), "database_name": materialized_view.get("database"), - "schedule": materialized_view.get("schedule"), + # "schedule": materialized_view.get("schedule"), "query": cls._parse_query(query.get("definition")), } @@ -147,18 +148,20 @@ class DatabricksMaterializedViewConfigChangeset: @property def requires_full_refresh(self) -> bool: - return any( - { - self.schedule.requires_full_refresh if self.schedule else False, - self.partition.requires_full_refresh if self.partition else False, - } - ) + return False + # return any( + # { + # self.schedule.requires_full_refresh if self.schedule else False, + # self.partition.requires_full_refresh if self.partition else False, + # } + # ) @property def has_changes(self) -> bool: - return any( - { - self.schedule if self.schedule else False, - self.partition if self.partition else False, - } - ) + return False + # return any( + # { + # self.schedule if self.schedule else False, + # self.partition if self.partition else False, + # } + # ) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py index 5af4d857b..3088ed814 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_views.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_views.py @@ -35,6 +35,8 @@ """ +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class TestDatabricksMaterializedViewsBasic(MaterializedViewBasic): @pytest.fixture(scope="class", autouse=True) def models(self): @@ -72,6 +74,8 @@ def test_materialized_view_create_idempotent(self, project, my_materialized_view assert self.query_relation_type(project, my_materialized_view) == "materialized_view" +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class DatabricksMaterializedViewChanges(MaterializedViewChanges): @pytest.fixture(scope="class", autouse=True) def models(self): @@ -120,9 +124,7 @@ def check_state_alter_change_is_applied_str_false(project, materialized_view): @staticmethod def change_config_via_replace(project, materialized_view): initial_model = get_model_file(project, materialized_view) - new_model = initial_model.replace("dist='id',", "").replace( - "sort=['id']", "sort=['value']" - ) + new_model = initial_model.replace("dist='id',", "").replace("sort=['id']", "sort=['value']") set_model_file(project, materialized_view, new_model) @staticmethod @@ -131,6 +133,8 @@ def check_state_replace_change_is_applied(project, materialized_view): assert query_dist(project, materialized_view) == "EVEN" +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class TestDatabricksMaterializedViewChangesApply( DatabricksMaterializedViewChanges, MaterializedViewChangesApplyMixin ): @@ -188,6 +192,8 @@ def test_change_is_applied_via_replace(self, project, my_materialized_view): assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs) +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class TestDatabricksMaterializedViewChangesContinue( DatabricksMaterializedViewChanges, MaterializedViewChangesContinueMixin ): @@ -229,6 +235,8 @@ def test_change_is_not_applied_via_replace(self, project, my_materialized_view): assert_message_in_logs(f"Applying REPLACE to: {my_materialized_view}", logs, False) +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class TestDatabricksMaterializedViewChangesFail( DatabricksMaterializedViewChanges, MaterializedViewChangesFailMixin ): @@ -245,6 +253,8 @@ class TestDatabricksMaterializedViewChangesFail( """ +@pytest.mark.skip_profile("databricks_cluster") +@pytest.mark.skip_profile("databricks_uc_cluster") class TestDatabricksMaterializedViewWithBackupConfig: @pytest.fixture(scope="class", autouse=True) def models(self): diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py deleted file mode 100644 index 443dc1830..000000000 --- a/tests/unit/relation_configs/test_materialized_view.py +++ /dev/null @@ -1,57 +0,0 @@ -from unittest.mock import Mock - -import pytest - -from dbt.adapters.databricks.relation_configs.materialized_view import ( - DatabricksMaterializedViewConfig, -) - - -@pytest.mark.parametrize("bool_value", [True, False, "True", "False", "true", "false"]) -def test_databricks_materialized_view_config_handles_all_valid_bools(bool_value): - config = DatabricksMaterializedViewConfig( - database_name="somedb", - schema_name="public", - mv_name="someview", - query="select * from sometable", - ) - model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" - ) - config_dict = config.parse_model_node(model_node) - assert isinstance(config_dict["autorefresh"], bool) - assert isinstance(config_dict["backup"], bool) - - -@pytest.mark.parametrize("bool_value", [1]) -def test_databricks_materialized_view_config_throws_expected_exception_with_invalid_types( - bool_value, -): - config = DatabricksMaterializedViewConfig( - database_name="somedb", - schema_name="public", - mv_name="someview", - query="select * from sometable", - ) - model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" - ) - with pytest.raises(TypeError): - config.parse_model_node(model_node) - - -def test_databricks_materialized_view_config_throws_expected_exception_with_invalid_str(): - config = DatabricksMaterializedViewConfig( - database_name="somedb", - schema_name="public", - mv_name="someview", - query="select * from sometable", - ) - model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" - ) - with pytest.raises(ValueError): - config.parse_model_node(model_node)