Merge branch 'main' into fix-microbatch-behavior-flag-conditional

.changes/unreleased/Fixes-20241112-143740.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Move require_batched_execution_for_custom_microbatch_strategy flag to global
+time: 2024-11-12T14:37:40.681284-06:00
+custom:
+  Author: QMalcolm MichelleArk
+  Issue: 351

dbt/adapters/base/impl.py

@@ -98,6 +98,13 @@
 GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations"
 FRESHNESS_MACRO_NAME = "collect_freshness"
 GET_RELATION_LAST_MODIFIED_MACRO_NAME = "get_relation_last_modified"
+DEFAULT_BASE_BEHAVIOR_FLAGS = [
+    {
+        "name": "require_batched_execution_for_custom_microbatch_strategy",
+        "default": False,
+        "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch",
+    }
+]
 
 
 class ConstraintSupport(str, Enum):
@@ -273,8 +280,7 @@ def __init__(self, config, mp_context: SpawnContext) -> None:
         self.connections = self.ConnectionManager(config, mp_context)
         self._macro_resolver: Optional[MacroResolverProtocol] = None
         self._macro_context_generator: Optional[MacroContextGeneratorCallable] = None
-        # this will be updated to include global behavior flags once they exist
-        self.behavior = []  # type: ignore
+        self.behavior = DEFAULT_BASE_BEHAVIOR_FLAGS  # type: ignore
 
     ###
     # Methods to set / access a macro resolver
@@ -314,14 +320,10 @@ def behavior(self, flags: List[BehaviorFlag]) -> None:
     def _behavior_flags(self) -> List[BehaviorFlag]:
         """
         This method should be overwritten by adapter maintainers to provide platform-specific flags
+
+        The BaseAdapter should NOT include any global flags here as those should be defined via DEFAULT_BASE_BEHAVIOR_FLAGS
         """
-        return [
-            {
-                "name": "require_batched_execution_for_custom_microbatch_strategy",
-                "default": False,
-                "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch",
-            }
-        ]
+        return []
 
     ###
     # Methods that pass through to the connection manager

tests/unit/behavior_flag_tests/test_behavior_flags.py

@@ -1,5 +1,6 @@
 from typing import Any, Dict, List
 
+from dbt.adapters.base.impl import DEFAULT_BASE_BEHAVIOR_FLAGS
 from dbt_common.behavior_flags import BehaviorFlag
 from dbt_common.exceptions import DbtBaseException
 import pytest
@@ -64,3 +65,12 @@ def test_register_behavior_flags(adapter):
     assert not adapter.behavior.default_true_user_false_flag
     assert adapter.behavior.default_true_user_true_flag
     assert adapter.behavior.default_true_user_skip_flag
+
+
+def test_behaviour_flags_property_empty(adapter_default_behaviour_flags):
+    assert adapter_default_behaviour_flags._behavior_flags == []
+
+
+def test_behavior_property_has_defaults(adapter_default_behaviour_flags):
+    for flag in DEFAULT_BASE_BEHAVIOR_FLAGS:
+        assert hasattr(adapter_default_behaviour_flags.behavior, flag["name"])

tests/unit/conftest.py

@@ -1 +1,7 @@
-from tests.unit.fixtures import adapter, behavior_flags, config, flags
+from tests.unit.fixtures import (
+    adapter,
+    adapter_default_behaviour_flags,
+    behavior_flags,
+    config,
+    flags,
+)

tests/unit/fixtures/__init__.py

@@ -1 +1,7 @@
-from tests.unit.fixtures.adapter import adapter, behavior_flags, config, flags
+from tests.unit.fixtures.adapter import (
+    adapter,
+    adapter_default_behaviour_flags,
+    behavior_flags,
+    config,
+    flags,
+)

tests/unit/fixtures/adapter.py

@@ -15,105 +15,110 @@
 from tests.unit.fixtures.credentials import CredentialsStub
 
 
-@pytest.fixture
-def adapter(config, behavior_flags) -> BaseAdapter:
+class BaseAdapterStub(BaseAdapter):
+    """
+    A stub for an adapter that uses the cache as the database
+    """
+
+    ConnectionManager = ConnectionManagerStub
+
+    ###
+    # Abstract methods for database-specific values, attributes, and types
+    ###
+    @classmethod
+    def date_function(cls) -> str:
+        return "date_function"
+
+    @classmethod
+    def is_cancelable(cls) -> bool:
+        return False
+
+    def list_schemas(self, database: str) -> List[str]:
+        return list(self.cache.schemas)
+
+    ###
+    # Abstract methods about relations
+    ###
+    def drop_relation(self, relation: BaseRelation) -> None:
+        self.cache_dropped(relation)
+
+    def truncate_relation(self, relation: BaseRelation) -> None:
+        self.cache_dropped(relation)
+
+    def rename_relation(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None:
+        self.cache_renamed(from_relation, to_relation)
+
+    def get_columns_in_relation(self, relation: BaseRelation) -> List[Column]:
+        # there's no database, so these need to be added as kwargs in the existing_relations fixture
+        return relation.columns
+
+    def expand_column_types(self, goal: BaseRelation, current: BaseRelation) -> None:
+        # there's no database, so these need to be added as kwargs in the existing_relations fixture
+        object.__setattr__(current, "columns", goal.columns)
+
+    def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[BaseRelation]:
+        # there's no database, so use the cache as the database
+        return self.cache.get_relations(schema_relation.database, schema_relation.schema)
+
+    ###
+    # ODBC FUNCTIONS -- these should not need to change for every adapter,
+    #                   although some adapters may override them
+    ###
+    def create_schema(self, relation: BaseRelation):
+        # there's no database, this happens implicitly by adding a relation to the cache
+        pass
+
+    def drop_schema(self, relation: BaseRelation):
+        for each_relation in self.cache.get_relations(relation.database, relation.schema):
+            self.cache_dropped(each_relation)
+
+    @classmethod
+    def quote(cls, identifier: str) -> str:
+        quote_char = ""
+        return f"{quote_char}{identifier}{quote_char}"
+
+    ###
+    # Conversions: These must be implemented by concrete implementations, for
+    # converting agate types into their sql equivalents.
+    ###
+    @classmethod
+    def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "str"
+
+    @classmethod
+    def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "float"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "bool"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "datetime"
+
+    @classmethod
+    def convert_date_type(cls, *args, **kwargs):
+        return "date"
+
+    @classmethod
+    def convert_time_type(cls, *args, **kwargs):
+        return "time"
 
-    class BaseAdapterStub(BaseAdapter):
-        """
-        A stub for an adapter that uses the cache as the database
-        """
 
-        ConnectionManager = ConnectionManagerStub
+@pytest.fixture
+def adapter(config, behavior_flags) -> BaseAdapter:
 
+    class BaseAdapterBehaviourFlagStub(BaseAdapterStub):
         @property
         def _behavior_flags(self) -> List[BehaviorFlag]:
             return behavior_flags
 
-        ###
-        # Abstract methods for database-specific values, attributes, and types
-        ###
-        @classmethod
-        def date_function(cls) -> str:
-            return "date_function"
-
-        @classmethod
-        def is_cancelable(cls) -> bool:
-            return False
-
-        def list_schemas(self, database: str) -> List[str]:
-            return list(self.cache.schemas)
-
-        ###
-        # Abstract methods about relations
-        ###
-        def drop_relation(self, relation: BaseRelation) -> None:
-            self.cache_dropped(relation)
-
-        def truncate_relation(self, relation: BaseRelation) -> None:
-            self.cache_dropped(relation)
-
-        def rename_relation(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None:
-            self.cache_renamed(from_relation, to_relation)
-
-        def get_columns_in_relation(self, relation: BaseRelation) -> List[Column]:
-            # there's no database, so these need to be added as kwargs in the existing_relations fixture
-            return relation.columns
-
-        def expand_column_types(self, goal: BaseRelation, current: BaseRelation) -> None:
-            # there's no database, so these need to be added as kwargs in the existing_relations fixture
-            object.__setattr__(current, "columns", goal.columns)
-
-        def list_relations_without_caching(
-            self, schema_relation: BaseRelation
-        ) -> List[BaseRelation]:
-            # there's no database, so use the cache as the database
-            return self.cache.get_relations(schema_relation.database, schema_relation.schema)
-
-        ###
-        # ODBC FUNCTIONS -- these should not need to change for every adapter,
-        #                   although some adapters may override them
-        ###
-        def create_schema(self, relation: BaseRelation):
-            # there's no database, this happens implicitly by adding a relation to the cache
-            pass
-
-        def drop_schema(self, relation: BaseRelation):
-            for each_relation in self.cache.get_relations(relation.database, relation.schema):
-                self.cache_dropped(each_relation)
-
-        @classmethod
-        def quote(cls, identifier: str) -> str:
-            quote_char = ""
-            return f"{quote_char}{identifier}{quote_char}"
-
-        ###
-        # Conversions: These must be implemented by concrete implementations, for
-        # converting agate types into their sql equivalents.
-        ###
-        @classmethod
-        def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-            return "str"
-
-        @classmethod
-        def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-            return "float"
-
-        @classmethod
-        def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-            return "bool"
-
-        @classmethod
-        def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-            return "datetime"
-
-        @classmethod
-        def convert_date_type(cls, *args, **kwargs):
-            return "date"
-
-        @classmethod
-        def convert_time_type(cls, *args, **kwargs):
-            return "time"
+    return BaseAdapterBehaviourFlagStub(config, get_context("spawn"))
+
 
+@pytest.fixture
+def adapter_default_behaviour_flags(config) -> BaseAdapter:
     return BaseAdapterStub(config, get_context("spawn"))