From 85122e591ffa6a0378abc736b8ea0a2162f11e09 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 13 Nov 2024 12:48:07 -0500 Subject: [PATCH] Fix setting of global behavior flags (#348) Co-authored-by: Quigley Malcolm --- .../unreleased/Fixes-20241112-143740.yaml | 6 + dbt/adapters/base/impl.py | 20 +- .../test_behavior_flags.py | 10 + tests/unit/conftest.py | 8 +- tests/unit/fixtures/__init__.py | 8 +- tests/unit/fixtures/adapter.py | 187 +++++++++--------- 6 files changed, 137 insertions(+), 102 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241112-143740.yaml diff --git a/.changes/unreleased/Fixes-20241112-143740.yaml b/.changes/unreleased/Fixes-20241112-143740.yaml new file mode 100644 index 00000000..ca899cbc --- /dev/null +++ b/.changes/unreleased/Fixes-20241112-143740.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Move require_batched_execution_for_custom_microbatch_strategy flag to global +time: 2024-11-12T14:37:40.681284-06:00 +custom: + Author: QMalcolm MichelleArk + Issue: 351 diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index c8457e2f..15093600 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -98,6 +98,13 @@ GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations" FRESHNESS_MACRO_NAME = "collect_freshness" GET_RELATION_LAST_MODIFIED_MACRO_NAME = "get_relation_last_modified" +DEFAULT_BASE_BEHAVIOR_FLAGS = [ + { + "name": "require_batched_execution_for_custom_microbatch_strategy", + "default": False, + "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch", + } +] class ConstraintSupport(str, Enum): @@ -273,8 +280,7 @@ def __init__(self, config, mp_context: SpawnContext) -> None: self.connections = self.ConnectionManager(config, mp_context) self._macro_resolver: Optional[MacroResolverProtocol] = None self._macro_context_generator: Optional[MacroContextGeneratorCallable] = None - # this will be updated to include global behavior flags once they exist - self.behavior = [] # type: ignore + self.behavior = DEFAULT_BASE_BEHAVIOR_FLAGS # type: ignore ### # Methods to set / access a macro resolver @@ -314,14 +320,10 @@ def behavior(self, flags: List[BehaviorFlag]) -> None: def _behavior_flags(self) -> List[BehaviorFlag]: """ This method should be overwritten by adapter maintainers to provide platform-specific flags + + The BaseAdapter should NOT include any global flags here as those should be defined via DEFAULT_BASE_BEHAVIOR_FLAGS """ - return [ - { - "name": "require_batched_execution_for_custom_microbatch_strategy", - "default": False, - "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch", - } - ] + return [] ### # Methods that pass through to the connection manager diff --git a/tests/unit/behavior_flag_tests/test_behavior_flags.py b/tests/unit/behavior_flag_tests/test_behavior_flags.py index 7f3abb89..378d07bb 100644 --- a/tests/unit/behavior_flag_tests/test_behavior_flags.py +++ b/tests/unit/behavior_flag_tests/test_behavior_flags.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +from dbt.adapters.base.impl import DEFAULT_BASE_BEHAVIOR_FLAGS from dbt_common.behavior_flags import BehaviorFlag from dbt_common.exceptions import DbtBaseException import pytest @@ -64,3 +65,12 @@ def test_register_behavior_flags(adapter): assert not adapter.behavior.default_true_user_false_flag assert adapter.behavior.default_true_user_true_flag assert adapter.behavior.default_true_user_skip_flag + + +def test_behaviour_flags_property_empty(adapter_default_behaviour_flags): + assert adapter_default_behaviour_flags._behavior_flags == [] + + +def test_behavior_property_has_defaults(adapter_default_behaviour_flags): + for flag in DEFAULT_BASE_BEHAVIOR_FLAGS: + assert hasattr(adapter_default_behaviour_flags.behavior, flag["name"]) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 346634df..225bdf57 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1 +1,7 @@ -from tests.unit.fixtures import adapter, behavior_flags, config, flags +from tests.unit.fixtures import ( + adapter, + adapter_default_behaviour_flags, + behavior_flags, + config, + flags, +) diff --git a/tests/unit/fixtures/__init__.py b/tests/unit/fixtures/__init__.py index 78135a2c..caa1448f 100644 --- a/tests/unit/fixtures/__init__.py +++ b/tests/unit/fixtures/__init__.py @@ -1 +1,7 @@ -from tests.unit.fixtures.adapter import adapter, behavior_flags, config, flags +from tests.unit.fixtures.adapter import ( + adapter, + adapter_default_behaviour_flags, + behavior_flags, + config, + flags, +) diff --git a/tests/unit/fixtures/adapter.py b/tests/unit/fixtures/adapter.py index b59b0423..3730a083 100644 --- a/tests/unit/fixtures/adapter.py +++ b/tests/unit/fixtures/adapter.py @@ -15,105 +15,110 @@ from tests.unit.fixtures.credentials import CredentialsStub -@pytest.fixture -def adapter(config, behavior_flags) -> BaseAdapter: +class BaseAdapterStub(BaseAdapter): + """ + A stub for an adapter that uses the cache as the database + """ + + ConnectionManager = ConnectionManagerStub + + ### + # Abstract methods for database-specific values, attributes, and types + ### + @classmethod + def date_function(cls) -> str: + return "date_function" + + @classmethod + def is_cancelable(cls) -> bool: + return False + + def list_schemas(self, database: str) -> List[str]: + return list(self.cache.schemas) + + ### + # Abstract methods about relations + ### + def drop_relation(self, relation: BaseRelation) -> None: + self.cache_dropped(relation) + + def truncate_relation(self, relation: BaseRelation) -> None: + self.cache_dropped(relation) + + def rename_relation(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None: + self.cache_renamed(from_relation, to_relation) + + def get_columns_in_relation(self, relation: BaseRelation) -> List[Column]: + # there's no database, so these need to be added as kwargs in the existing_relations fixture + return relation.columns + + def expand_column_types(self, goal: BaseRelation, current: BaseRelation) -> None: + # there's no database, so these need to be added as kwargs in the existing_relations fixture + object.__setattr__(current, "columns", goal.columns) + + def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[BaseRelation]: + # there's no database, so use the cache as the database + return self.cache.get_relations(schema_relation.database, schema_relation.schema) + + ### + # ODBC FUNCTIONS -- these should not need to change for every adapter, + # although some adapters may override them + ### + def create_schema(self, relation: BaseRelation): + # there's no database, this happens implicitly by adding a relation to the cache + pass + + def drop_schema(self, relation: BaseRelation): + for each_relation in self.cache.get_relations(relation.database, relation.schema): + self.cache_dropped(each_relation) + + @classmethod + def quote(cls, identifier: str) -> str: + quote_char = "" + return f"{quote_char}{identifier}{quote_char}" + + ### + # Conversions: These must be implemented by concrete implementations, for + # converting agate types into their sql equivalents. + ### + @classmethod + def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "str" + + @classmethod + def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "float" + + @classmethod + def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "bool" + + @classmethod + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "datetime" + + @classmethod + def convert_date_type(cls, *args, **kwargs): + return "date" + + @classmethod + def convert_time_type(cls, *args, **kwargs): + return "time" - class BaseAdapterStub(BaseAdapter): - """ - A stub for an adapter that uses the cache as the database - """ - ConnectionManager = ConnectionManagerStub +@pytest.fixture +def adapter(config, behavior_flags) -> BaseAdapter: + class BaseAdapterBehaviourFlagStub(BaseAdapterStub): @property def _behavior_flags(self) -> List[BehaviorFlag]: return behavior_flags - ### - # Abstract methods for database-specific values, attributes, and types - ### - @classmethod - def date_function(cls) -> str: - return "date_function" - - @classmethod - def is_cancelable(cls) -> bool: - return False - - def list_schemas(self, database: str) -> List[str]: - return list(self.cache.schemas) - - ### - # Abstract methods about relations - ### - def drop_relation(self, relation: BaseRelation) -> None: - self.cache_dropped(relation) - - def truncate_relation(self, relation: BaseRelation) -> None: - self.cache_dropped(relation) - - def rename_relation(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None: - self.cache_renamed(from_relation, to_relation) - - def get_columns_in_relation(self, relation: BaseRelation) -> List[Column]: - # there's no database, so these need to be added as kwargs in the existing_relations fixture - return relation.columns - - def expand_column_types(self, goal: BaseRelation, current: BaseRelation) -> None: - # there's no database, so these need to be added as kwargs in the existing_relations fixture - object.__setattr__(current, "columns", goal.columns) - - def list_relations_without_caching( - self, schema_relation: BaseRelation - ) -> List[BaseRelation]: - # there's no database, so use the cache as the database - return self.cache.get_relations(schema_relation.database, schema_relation.schema) - - ### - # ODBC FUNCTIONS -- these should not need to change for every adapter, - # although some adapters may override them - ### - def create_schema(self, relation: BaseRelation): - # there's no database, this happens implicitly by adding a relation to the cache - pass - - def drop_schema(self, relation: BaseRelation): - for each_relation in self.cache.get_relations(relation.database, relation.schema): - self.cache_dropped(each_relation) - - @classmethod - def quote(cls, identifier: str) -> str: - quote_char = "" - return f"{quote_char}{identifier}{quote_char}" - - ### - # Conversions: These must be implemented by concrete implementations, for - # converting agate types into their sql equivalents. - ### - @classmethod - def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "str" - - @classmethod - def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "float" - - @classmethod - def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "bool" - - @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: - return "datetime" - - @classmethod - def convert_date_type(cls, *args, **kwargs): - return "date" - - @classmethod - def convert_time_type(cls, *args, **kwargs): - return "time" + return BaseAdapterBehaviourFlagStub(config, get_context("spawn")) + +@pytest.fixture +def adapter_default_behaviour_flags(config) -> BaseAdapter: return BaseAdapterStub(config, get_context("spawn"))