Relation/Materialization Prep Reformat

.changes/unreleased/Under the Hood-20230718-200942.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Refactor jinja templates and relation models in support of new relation framework
+time: 2023-07-18T20:09:42.008424-04:00
+custom:
+  Author: mikealfare
+  Issue: "8128"

.flake8

@@ -10,3 +10,5 @@ ignore =
     E741
     E501 # long line checking is done in black
 exclude = test/
+per-file-ignores =
+    */__init__.py: F401

core/dbt/adapters/base/impl.py

@@ -1,5 +1,5 @@
 import abc
-from concurrent.futures import as_completed, Future
+from concurrent.futures import Future, as_completed
 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum
@@ -20,11 +20,36 @@
     Union,
 )
 
-from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
-
 import agate
 import pytz
 
+from dbt import deprecations
+from dbt.adapters.base import Column as BaseColumn, Credentials
+from dbt.adapters.base.connections import AdapterResponse, Connection
+from dbt.adapters.base.meta import AdapterMeta, available
+from dbt.adapters.base.relation import BaseRelation, InformationSchema, SchemaSearchMap
+from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
+from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol
+from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows
+from dbt.clients.jinja import MacroGenerator
+from dbt.contracts.graph.manifest import Manifest, MacroManifest
+from dbt.contracts.graph.nodes import (
+    ColumnLevelConstraint,
+    ConstraintType,
+    ModelLevelConstraint,
+    ResultNode,
+)
+from dbt.contracts.relation import ComponentName
+from dbt.events.functions import fire_event, warn_or_error
+from dbt.events.types import (
+    CacheMiss,
+    CatalogGenerationError,
+    CodeExecution,
+    CodeExecutionStatus,
+    ConstraintNotEnforced,
+    ConstraintNotSupported,
+    ListRelations,
+)
 from dbt.exceptions import (
     DbtInternalError,
     DbtRuntimeError,
@@ -42,36 +67,8 @@
     UnexpectedNonTimestampError,
     UnexpectedNullError,
 )
-
-from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol
-from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows
-from dbt.clients.jinja import MacroGenerator
-from dbt.contracts.graph.manifest import Manifest, MacroManifest
-from dbt.contracts.graph.nodes import ResultNode
-from dbt.events.functions import fire_event, warn_or_error
-from dbt.events.types import (
-    CacheMiss,
-    ListRelations,
-    CodeExecution,
-    CodeExecutionStatus,
-    CatalogGenerationError,
-    ConstraintNotSupported,
-    ConstraintNotEnforced,
-)
 from dbt.utils import filter_null_values, executor, cast_to_str, AttrDict
 
-from dbt.adapters.base.connections import Connection, AdapterResponse
-from dbt.adapters.base.meta import AdapterMeta, available
-from dbt.adapters.base.relation import (
-    ComponentName,
-    BaseRelation,
-    InformationSchema,
-    SchemaSearchMap,
-)
-from dbt.adapters.base import Column as BaseColumn
-from dbt.adapters.base import Credentials
-from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
-from dbt import deprecations
 
 GET_CATALOG_MACRO_NAME = "get_catalog"
 FRESHNESS_MACRO_NAME = "collect_freshness"
@@ -857,7 +854,7 @@ def quote(cls, identifier: str) -> str:
 
     @available
     def quote_as_configured(self, identifier: str, quote_key: str) -> str:
-        """Quote or do not quote the given identifer as configured in the
+        """Quote or do not quote the given identifier as configured in the
         project config for the quote key.
 
         The quote key should be one of 'database' (on bigquery, 'profile'),

core/dbt/adapters/base/relation.py

@@ -1,25 +1,24 @@
 from collections.abc import Hashable
 from dataclasses import dataclass, field
-from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set
+from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, TypeVar
 
-from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode
+from dbt.contracts.graph.nodes import ManifestNode, ParsedNode, ResultNode, SourceDefinition
 from dbt.contracts.relation import (
-    RelationType,
     ComponentName,
-    HasQuoting,
     FakeAPIObject,
-    Policy,
+    HasQuoting,
     Path,
+    Policy,
+    RelationType,
 )
+import dbt.exceptions
 from dbt.exceptions import (
     ApproximateMatchError,
     DbtInternalError,
     MultipleDatabasesNotAllowedError,
 )
 from dbt.node_types import NodeType
-from dbt.utils import filter_null_values, deep_merge, classproperty
-
-import dbt.exceptions
+from dbt.utils import classproperty, deep_merge, filter_null_values
 
 
 Self = TypeVar("Self", bound="BaseRelation")
@@ -31,7 +30,8 @@ class BaseRelation(FakeAPIObject, Hashable):
     type: Optional[RelationType] = None
     quote_character: str = '"'
     # Python 3.11 requires that these use default_factory instead of simple default
-    # ValueError: mutable default <class 'dbt.contracts.relation.Policy'> for field include_policy is not allowed: use default_factory
+    # ValueError: mutable default <class 'dbt.contracts.relation.Policy'> for field include_policy is not allowed:
+    # use default_factory
     include_policy: Policy = field(default_factory=lambda: Policy())
     quote_policy: Policy = field(default_factory=lambda: Policy())
     dbt_created: bool = False
@@ -170,13 +170,14 @@ def without_identifier(self) -> "BaseRelation":
 
     def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[str]]]:
 
-        for key in ComponentName:
+        for member in ComponentName:
             path_part: Optional[str] = None
-            if self.include_policy.get_part(key):
-                path_part = self.path.get_part(key)
-                if path_part is not None and self.quote_policy.get_part(key):
+            component = ComponentName(member)
+            if self.include_policy.get_part(component):
+                path_part = self.path.get_part(component)
+                if path_part is not None and self.quote_policy.get_part(component):
                     path_part = self.quoted(path_part)
-            yield key, path_part
+            yield component, path_part
 
     def render(self) -> str:
         # if there is nothing set, this will return the empty string.
@@ -259,7 +260,7 @@ def create_from(
             return cls.create_from_source(node, **kwargs)
         else:
             # Can't use ManifestNode here because of parameterized generics
-            if not isinstance(node, (ParsedNode)):
+            if not isinstance(node, ParsedNode):
                 raise DbtInternalError(
                     f"type mismatch, expected ManifestNode but got {type(node)}"
                 )

core/dbt/adapters/relation/models/__init__.py

@@ -0,0 +1,8 @@
+from dbt.adapters.relation.models._relation import (
+    DescribeRelationResults,
+    RelationComponent,
+)
+from dbt.adapters.relation.models._change import (
+    RelationChangeAction,
+    RelationChange,
+)

core/dbt/adapters/relation_configs/config_change.py → core/dbt/adapters/relation/models/_change.py

@@ -2,19 +2,19 @@
 from dataclasses import dataclass
 from typing import Hashable
 
-from dbt.adapters.relation_configs.config_base import RelationConfigBase
+from dbt.adapters.relation.models._relation import RelationComponent
 from dbt.dataclass_schema import StrEnum
 
 
-class RelationConfigChangeAction(StrEnum):
+class RelationChangeAction(StrEnum):
     alter = "alter"
     create = "create"
     drop = "drop"
 
 
 @dataclass(frozen=True, eq=True, unsafe_hash=True)
-class RelationConfigChange(RelationConfigBase, ABC):
-    action: RelationConfigChangeAction
+class RelationChange(RelationComponent, ABC):
+    action: RelationChangeAction
     context: Hashable  # this is usually a RelationConfig, e.g. IndexConfig, but shouldn't be limited
 
     @property

core/dbt/adapters/relation_configs/config_base.py → core/dbt/adapters/relation/models/_relation.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-from typing import Union, Dict
+from typing import Any, Dict, Union
 
 import agate
+
 from dbt.utils import filter_null_values
 
 
@@ -18,24 +19,24 @@
     ])
 }
 """
-RelationResults = Dict[str, Union[agate.Row, agate.Table]]
+DescribeRelationResults = Dict[str, Union[agate.Row, agate.Table]]
 
 
 @dataclass(frozen=True)
-class RelationConfigBase:
+class RelationComponent:
     @classmethod
-    def from_dict(cls, kwargs_dict) -> "RelationConfigBase":
+    def from_dict(cls, config_dict: Dict[str, Any]) -> "RelationComponent":
         """
         This assumes the subclass of `RelationConfigBase` is flat, in the sense that no attribute is
         itself another subclass of `RelationConfigBase`. If that's not the case, this should be overriden
         to manually manage that complexity.
 
         Args:
-            kwargs_dict: the dict representation of this instance
+            config_dict: the dict representation of this instance
 
         Returns: the `RelationConfigBase` representation associated with the provided dict
         """
-        return cls(**filter_null_values(kwargs_dict))  # type: ignore
+        return cls(**filter_null_values(config_dict))  # type: ignore
 
     @classmethod
     def _not_implemented_error(cls) -> NotImplementedError:

core/dbt/adapters/sql/impl.py

@@ -1,17 +1,16 @@
 import agate
-from typing import Any, Optional, Tuple, Type, List
+from typing import Any, List, Optional, Tuple, Type
 
-from dbt.contracts.connection import Connection, AdapterResponse
-from dbt.exceptions import RelationTypeNullError
 from dbt.adapters.base import BaseAdapter, available
+from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.cache import _make_ref_key_dict
 from dbt.adapters.sql import SQLConnectionManager
+from dbt.contracts.connection import AdapterResponse, Connection
 from dbt.events.functions import fire_event
 from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+from dbt.exceptions import RelationTypeNullError
 
 
-from dbt.adapters.base.relation import BaseRelation
-
 LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching"
 GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation"
 LIST_SCHEMAS_MACRO_NAME = "list_schemas"
@@ -222,14 +221,14 @@ def check_schema_exists(self, database: str, schema: str) -> bool:
     def validate_sql(self, sql: str) -> AdapterResponse:
         """Submit the given SQL to the engine for validation, but not execution.
 
-        By default we simply prefix the query with the explain keyword and allow the
+        By default, we simply prefix the query with the explain keyword and allow the
         exceptions thrown by the underlying engine on invalid SQL inputs to bubble up
         to the exception handler. For adjustments to the explain statement - such as
         for adapters that have different mechanisms for hinting at query validation
         or dry-run - callers may be able to override the validate_sql_query macro with
         the addition of an <adapter>__validate_sql implementation.
 
-        :param sql str: The sql to validate
+        :param sql: str The sql to validate
         """
         kwargs = {
             "sql": sql,

core/dbt/adapters/relation_configs/config_validation.py → core/dbt/adapters/validation.py

@@ -5,7 +5,7 @@
 
 
 @dataclass(frozen=True, eq=True, unsafe_hash=True)
-class RelationConfigValidationRule:
+class ValidationRule:
     validation_check: bool
     validation_error: Optional[DbtRuntimeError]
 
@@ -18,12 +18,12 @@ def default_error(self):
 
 
 @dataclass(frozen=True)
-class RelationConfigValidationMixin:
+class ValidationMixin:
     def __post_init__(self):
         self.run_validation_rules()
 
     @property
-    def validation_rules(self) -> Set[RelationConfigValidationRule]:
+    def validation_rules(self) -> Set[ValidationRule]:
         """
         A set of validation rules to run against the object upon creation.