Merge branch 'main' into snapshot_timestamp_data_types

.changes/unreleased/Breaking Changes-20231206-192442.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Fix changing the current working directory when using dpt deps, clean and init.
+time: 2023-12-06T19:24:42.575372+09:00
+custom:
+  Author: rariyama
+  Issue: "8997"

.changes/unreleased/Fixes-20240816-140807.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Filter out empty nodes after graph selection to support consistent selection of nodes that depend on upstream public models
+time: 2024-08-16T14:08:07.426235-07:00
+custom:
+  Author: jtcohen6
+  Issue: "8987"

.changes/unreleased/Fixes-20240824-210903.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Late render pre- and post-hooks configs in properties / schema YAML files
+time: 2024-08-24T21:09:03.252733-06:00
+custom:
+  Author: dbeatty10
+  Issue: "10603"

.changes/unreleased/Fixes-20240829-105701.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Allow the use of env_var function in certain macros in which it was previously
+  unavailable.
+time: 2024-08-29T10:57:01.160613-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "10609"

.changes/unreleased/Under the Hood-20240827-105014.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Additional type hints for `core/dbt/version.py`
+time: 2024-08-27T10:50:14.047859-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10612"

.changes/unreleased/Under the Hood-20240827-113123.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Fix typing issues in core/dbt/contracts/sql.py
+time: 2024-08-27T11:31:23.749912-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10614"

.changes/unreleased/Under the Hood-20240827-114810.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Fix type errors in `dbt/core/task/clean.py`
+time: 2024-08-27T11:48:10.438173-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10616"

core/dbt/cli/exceptions.py

@@ -1,7 +1,10 @@
-from typing import IO, Optional
+from typing import IO, List, Optional, Union
 
 from click.exceptions import ClickException
 
+from dbt.artifacts.schemas.catalog import CatalogArtifact
+from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.results import RunExecutionResult
 from dbt.utils import ExitCodes
 
 
@@ -23,15 +26,25 @@ def __init__(self, exit_code: ExitCodes) -> None:
 
     # the typing of _file is to satisfy the signature of ClickException.show
     # overriding this method prevents click from printing any exceptions to stdout
-    def show(self, _file: Optional[IO] = None) -> None:
+    def show(self, _file: Optional[IO] = None) -> None:  # type: ignore[type-arg]
         pass
 
 
 class ResultExit(CliException):
     """This class wraps any exception that contains results while invoking dbt, or the
     results of an invocation that did not succeed but did not throw any exceptions."""
 
-    def __init__(self, result) -> None:
+    def __init__(
+        self,
+        result: Union[
+            bool,  # debug
+            CatalogArtifact,  # docs generate
+            List[str],  # list/ls
+            Manifest,  # parse
+            None,  # clean, deps, init, source
+            RunExecutionResult,  # build, compile, run, seed, snapshot, test, run-operation
+        ] = None,
+    ) -> None:
         super().__init__(ExitCodes.ModelError)
         self.result = result
 

core/dbt/cli/main.py

@@ -218,10 +218,9 @@ def clean(ctx, **kwargs):
     """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)"""
     from dbt.task.clean import CleanTask
 
-    task = CleanTask(ctx.obj["flags"], ctx.obj["project"])
-
-    results = task.run()
-    success = task.interpret_results(results)
+    with CleanTask(ctx.obj["flags"], ctx.obj["project"]) as task:
+        results = task.run()
+        success = task.interpret_results(results)
     return results, success
 
 
@@ -437,9 +436,9 @@ def deps(ctx, **kwargs):
                 message=f"Version is required in --add-package when a package when source is {flags.SOURCE}",
                 option_name="--add-package",
             )
-    task = DepsTask(flags, ctx.obj["project"])
-    results = task.run()
-    success = task.interpret_results(results)
+    with DepsTask(flags, ctx.obj["project"]) as task:
+        results = task.run()
+        success = task.interpret_results(results)
     return results, success
 
 
@@ -459,10 +458,9 @@ def init(ctx, **kwargs):
     """Initialize a new dbt project."""
     from dbt.task.init import InitTask
 
-    task = InitTask(ctx.obj["flags"])
-
-    results = task.run()
-    success = task.interpret_results(results)
+    with InitTask(ctx.obj["flags"]) as task:
+        results = task.run()
+        success = task.interpret_results(results)
     return results, success
 
 

core/dbt/compilation.py

@@ -547,6 +547,8 @@ def compile_node(
         the node's raw_code into compiled_code, and then calls the
         recursive method to "prepend" the ctes.
         """
+        # REVIEW: UnitTestDefinition shouldn't be possible here because of the
+        # type of node, and it is likewise an invalid return type.
         if isinstance(node, UnitTestDefinition):
             return node
 

core/dbt/contracts/sql.py

@@ -29,7 +29,8 @@ class RemoteCompileResult(RemoteCompileResultMixin):
     generated_at: datetime = field(default_factory=datetime.utcnow)
 
     @property
-    def error(self):
+    def error(self) -> None:
+        # TODO: Can we delete this? It's never set anywhere else and never accessed
         return None
 
 
@@ -40,7 +41,7 @@ class RemoteExecutionResult(ExecutionResult):
     args: Dict[str, Any] = field(default_factory=dict)
     generated_at: datetime = field(default_factory=datetime.utcnow)
 
-    def write(self, path: str):
+    def write(self, path: str) -> None:
         writable = RunResultsArtifact.from_execution_results(
             generated_at=self.generated_at,
             results=self.results,

core/dbt/deprecations.py

@@ -138,7 +138,7 @@ def cb():
     return cb
 
 
-def warn(name, *args, **kwargs):
+def warn(name: str, *args, **kwargs) -> None:
     if name not in deprecations:
         # this should (hopefully) never happen
         raise RuntimeError("Error showing deprecation warning: {}".format(name))

core/dbt/graph/selector.py

@@ -87,12 +87,15 @@ def get_nodes_from_criteria(
             )
             return set(), set()
 
+        neighbors = self.collect_specified_neighbors(spec, collected)
+        selected = collected | neighbors
+
+        # if --indirect-selection EMPTY, do not expand to adjacent tests
         if spec.indirect_selection == IndirectSelection.Empty:
-            return collected, set()
+            return selected, set()
         else:
-            neighbors = self.collect_specified_neighbors(spec, collected)
             direct_nodes, indirect_nodes = self.expand_selection(
-                selected=(collected | neighbors), indirect_selection=spec.indirect_selection
+                selected=selected, indirect_selection=spec.indirect_selection
             )
             return direct_nodes, indirect_nodes
 
@@ -177,10 +180,14 @@ def _is_graph_member(self, unique_id: UniqueId) -> bool:
 
         node = self.manifest.nodes[unique_id]
 
-        if self.include_empty_nodes:
-            return node.config.enabled
+        return node.config.enabled
+
+    def _is_empty_node(self, unique_id: UniqueId) -> bool:
+        if unique_id in self.manifest.nodes:
+            node = self.manifest.nodes[unique_id]
+            return node.empty
         else:
-            return not node.empty and node.config.enabled
+            return False
 
     def node_is_match(self, node: GraphMemberNode) -> bool:
         """Determine if a node is a match for the selector. Non-match nodes
@@ -212,7 +219,12 @@ def filter_selection(self, selected: Set[UniqueId]) -> Set[UniqueId]:
         """Return the subset of selected nodes that is a match for this
         selector.
         """
-        return {unique_id for unique_id in selected if self._is_match(unique_id)}
+        return {
+            unique_id
+            for unique_id in selected
+            if self._is_match(unique_id)
+            and (self.include_empty_nodes or not self._is_empty_node(unique_id))
+        }
 
     def expand_selection(
         self,

core/dbt/parser/manifest.py

@@ -1028,12 +1028,11 @@ def build_manifest_state_check(self):
         return state_check
 
     def save_macros_to_adapter(self, adapter):
-        macro_manifest = MacroManifest(self.manifest.macros)
-        adapter.set_macro_resolver(macro_manifest)
+        adapter.set_macro_resolver(self.manifest)
         # This executes the callable macro_hook and sets the
         # query headers
         # This executes the callable macro_hook and sets the query headers
-        query_header_context = generate_query_header_context(adapter.config, macro_manifest)
+        query_header_context = generate_query_header_context(adapter.config, self.manifest)
         self.macro_hook(query_header_context)
 
     # This creates a MacroManifest which contains the macros in

core/dbt/parser/schema_renderer.py

@@ -11,6 +11,7 @@
 # keyword args are rendered to capture refs in render_test_update.
 # Keyword args are finally rendered at compilation time.
 # Descriptions are not rendered until 'process_docs'.
+# Pre- and post-hooks in configs are late-rendered.
 class SchemaYamlRenderer(BaseRenderer):
     def __init__(self, context: Dict[str, Any], key: str) -> None:
         super().__init__(context)
@@ -43,6 +44,14 @@ def _is_norender_key(self, keypath: Keypath) -> bool:
         if len(keypath) == 2 and keypath[1] in ("tests", "data_tests", "description"):
             return True
 
+        # pre- and post-hooks
+        if (
+            len(keypath) >= 2
+            and keypath[0] == "config"
+            and keypath[1] in ("pre_hook", "post_hook")
+        ):
+            return True
+
         # versions
         if len(keypath) == 5 and keypath[4] == "description":
             return True

core/dbt/plugins/manifest.py

@@ -17,5 +17,5 @@ class PluginNodes:
     def add_model(self, model_args: ModelNodeArgs) -> None:
         self.models[model_args.unique_id] = model_args
 
-    def update(self, other: "PluginNodes"):
+    def update(self, other: "PluginNodes") -> None:
         self.models.update(other.models)