Merge branch 'main' into foreign-ref-column-constraint

.changes/unreleased/Under the Hood-20240528-110518.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Allow dynamic selection of record types when recording.
+time: 2024-05-28T11:05:18.290107-05:00
+custom:
+  Author: emmyoop
+  Issue: "140"

.changes/unreleased/Under the Hood-20240529-143154.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move StatsItem, StatsDict, TableMetadata to dbt-common
+time: 2024-05-29T14:31:54.854468+01:00
+custom:
+    Author: aranke
+    Issue: "141"

.changes/unreleased/Under the Hood-20240603-123631.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move CatalogKey, ColumnMetadata, ColumnMap, CatalogTable to dbt-common
+time: 2024-06-03T12:36:31.542118+02:00
+custom:
+    Author: aranke
+    Issue: "147"

.changes/unreleased/Under the Hood-20240617-204541.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add support for basic diff of run recordings.
+time: 2024-06-17T20:45:41.123374-05:00
+custom:
+  Author: emmyoop
+  Issue: "144"

.changes/unreleased/Under the Hood-20240618-155025.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Deserialize Record objects on a just-in-time basis.
+time: 2024-06-18T15:50:25.985387-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "151"

.gitignore

@@ -157,4 +157,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.1.0"
+version = "1.5.0"

dbt_common/clients/system.py

@@ -62,7 +62,9 @@ def _include(self) -> bool:
         # Do not record or replay filesystem searches that were performed against
         # files which are actually part of dbt's implementation.
         return (
-            "dbt/include/global_project" not in self.root_path
+            "dbt/include"
+            not in self.root_path  # TODO: This actually obviates the next two checks but is probably too coarse?
+            and "dbt/include/global_project" not in self.root_path
             and "/plugins/postgres/dbt/include/" not in self.root_path
         )
 

dbt_common/context.py

@@ -2,14 +2,15 @@
 from typing import List, Mapping, Optional
 
 from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
+from dbt_common.record import Recorder
 
 
 class InvocationContext:
     def __init__(self, env: Mapping[str, str]):
         self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)}
         self._env_secrets: Optional[List[str]] = None
         self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)}
-        self.recorder = None
+        self.recorder: Optional[Recorder] = None
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 
     @property
@@ -32,7 +33,7 @@ def env_secrets(self) -> List[str]:
 _INVOCATION_CONTEXT_VAR: ContextVar[InvocationContext] = ContextVar("DBT_INVOCATION_CONTEXT_VAR")
 
 
-def reliably_get_invocation_var() -> ContextVar:
+def reliably_get_invocation_var() -> ContextVar[InvocationContext]:
     invocation_var: Optional[ContextVar] = next(
         (cv for cv in copy_context() if cv.name == _INVOCATION_CONTEXT_VAR.name), None
     )

dbt_common/contracts/constraints.py

@@ -39,27 +39,9 @@ class ColumnLevelConstraint(dbtClassMixin):
     to: Optional[str] = None
     to_column: Optional[str] = None
 
-    @classmethod
-    def validate(cls, data):
-        super().validate(data)
-        if data.get("type") is not ConstraintType.foreign_key:
-            if data.get("to") is not None:
-                raise ValidationError(f"Only column-level constraint of type {ConstraintType.foreign_key} can specify a 'to' field.")
-            if data.get("to_column") is not None:
-                raise ValidationError(f"Only column-level constraint of type {ConstraintType.foreign_key} can specify a 'to_column' field.")
-
 
 @dataclass
 class ModelLevelConstraint(ColumnLevelConstraint):
     columns: List[str] = field(default_factory=list)
     to: Optional[str] = None
     to_columns: List[str] = field(default_factory=list)
-
-    @classmethod
-    def validate(cls, data):
-        super().validate(data)
-        if data.get("type") is not ConstraintType.foreign_key:
-            if data.get("to") is not None:
-                raise ValidationError(f"Only model-level constraint of type {ConstraintType.foreign_key} can specify a 'to' field.")
-            if data.get("to_columns") is not None:
-                raise ValidationError(f"Only model-level constraint of type {ConstraintType.foreign_key} can specify a 'to_columns' field.")

dbt_common/contracts/metadata.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from typing import Dict, Optional, Union, NamedTuple
+
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.utils.formatting import lowercase
+
+
+@dataclass
+class StatsItem(dbtClassMixin):
+    id: str
+    label: str
+    value: Union[bool, str, float, None]
+    include: bool
+    description: Optional[str] = None
+
+
+StatsDict = Dict[str, StatsItem]
+
+
+@dataclass
+class TableMetadata(dbtClassMixin):
+    type: str
+    schema: str
+    name: str
+    database: Optional[str] = None
+    comment: Optional[str] = None
+    owner: Optional[str] = None
+
+
+CatalogKey = NamedTuple(
+    "CatalogKey", [("database", Optional[str]), ("schema", str), ("name", str)]
+)
+
+
+@dataclass
+class ColumnMetadata(dbtClassMixin):
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+ColumnMap = Dict[str, ColumnMetadata]
+
+
+@dataclass
+class CatalogTable(dbtClassMixin):
+    metadata: TableMetadata
+    columns: ColumnMap
+    stats: StatsDict
+    # the same table with two unique IDs will just be listed two times
+    unique_id: Optional[str] = None
+
+    def key(self) -> CatalogKey:
+        return CatalogKey(
+            lowercase(self.metadata.database),
+            self.metadata.schema.lower(),
+            self.metadata.name.lower(),
+        )

dbt_common/dataclass_schema.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, cast, get_type_hints, List, Tuple, Dict, Any, Optional
+from typing import Any, cast, ClassVar, Dict, get_type_hints, List, Optional, Tuple
 import re
 import jsonschema
 from dataclasses import fields, Field
@@ -26,7 +26,7 @@ class ValidationError(jsonschema.ValidationError):
 
 
 class DateTimeSerialization(SerializationStrategy):
-    def serialize(self, value) -> str:
+    def serialize(self, value: datetime) -> str:
         out = value.isoformat()
         # Assume UTC if timezone is missing
         if value.tzinfo is None:
@@ -127,7 +127,7 @@ def _get_fields(cls) -> List[Tuple[Field, str]]:
 
     # copied from hologram. Used in tests
     @classmethod
-    def _get_field_names(cls):
+    def _get_field_names(cls) -> List[str]:
         return [element[1] for element in cls._get_fields()]
 
 
@@ -152,7 +152,7 @@ def validate(cls, value):
 
 # These classes must be in this order or it doesn't work
 class StrEnum(str, SerializableType, Enum):
-    def __str__(self):
+    def __str__(self) -> str:
         return self.value
 
     # https://docs.python.org/3.6/library/enum.html#using-automatic-values

dbt_common/exceptions/base.py

@@ -1,5 +1,5 @@
 import builtins
-from typing import List, Any, Optional
+from typing import Any, List, Optional
 import os
 
 from dbt_common.constants import SECRET_ENV_PREFIX
@@ -37,7 +37,7 @@ def __init__(self, msg: str):
         self.msg = scrub_secrets(msg, env_secrets())
 
     @property
-    def type(self):
+    def type(self) -> str:
         return "Internal"
 
     def process_stack(self):
@@ -59,7 +59,7 @@ def process_stack(self):
 
         return lines
 
-    def __str__(self):
+    def __str__(self) -> str:
         if hasattr(self.msg, "split"):
             split_msg = self.msg.split("\n")
         else:

dbt_common/helper_types.py

@@ -19,7 +19,7 @@
 class NVEnum(StrEnum):
     novalue = "novalue"
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return isinstance(other, NVEnum)
 
 
@@ -59,7 +59,7 @@ def includes(self, item_name: str) -> bool:
             item_name in self.include or self.include in self.INCLUDE_ALL
         ) and item_name not in self.exclude
 
-    def _validate_items(self, items: List[str]):
+    def _validate_items(self, items: List[str]) -> None:
         pass