Merge branch 'main' into catalogtable_dbt_common

.changes/unreleased/Features-20240531-150816.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Update data_test to accept arbitrary config options
+time: 2024-05-31T15:08:16.431966-05:00
+custom:
+    Author: McKnight-42
+    Issue: "10197"

.changes/unreleased/Fixes-20240113-073615.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Convert "Skipping model due to fail_fast" message to DEBUG level
+time: 2024-01-13T07:36:15.836294-00:00
+custom:
+  Author: scottgigante,nevdelap
+  Issue: "8774"

.changes/unreleased/Fixes-20240605-111652.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix issues with selectors and inline nodes
+time: 2024-06-05T11:16:52.187667-04:00
+custom:
+  Author: gshank
+  Issue: 8943 9269

.changes/unreleased/Fixes-20240607-134648.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix snapshot config to work in yaml files
+time: 2024-06-07T13:46:48.383215-04:00
+custom:
+  Author: gshank
+  Issue: "4000"

core/dbt/artifacts/resources/v1/snapshot.py

@@ -18,39 +18,35 @@ class SnapshotConfig(NodeConfig):
     # Not using Optional because of serialization issues with a Union of str and List[str]
     check_cols: Union[str, List[str], None] = None
 
-    @classmethod
-    def validate(cls, data):
-        super().validate(data)
-        # Note: currently you can't just set these keys in schema.yml because this validation
-        # will fail when parsing the snapshot node.
-        if not data.get("strategy") or not data.get("unique_key") or not data.get("target_schema"):
+    def final_validate(self):
+        if not self.strategy or not self.unique_key or not self.target_schema:
             raise ValidationError(
                 "Snapshots must be configured with a 'strategy', 'unique_key', "
                 "and 'target_schema'."
             )
-        if data.get("strategy") == "check":
-            if not data.get("check_cols"):
+        if self.strategy == "check":
+            if not self.check_cols:
                 raise ValidationError(
                     "A snapshot configured with the check strategy must "
                     "specify a check_cols configuration."
                 )
-            if isinstance(data["check_cols"], str) and data["check_cols"] != "all":
+            if isinstance(self.check_cols, str) and self.check_cols != "all":
                 raise ValidationError(
-                    f"Invalid value for 'check_cols': {data['check_cols']}. "
+                    f"Invalid value for 'check_cols': {self.check_cols}. "
                     "Expected 'all' or a list of strings."
                 )
-        elif data.get("strategy") == "timestamp":
-            if not data.get("updated_at"):
+        elif self.strategy == "timestamp":
+            if not self.updated_at:
                 raise ValidationError(
                     "A snapshot configured with the timestamp strategy "
                     "must specify an updated_at configuration."
                 )
-            if data.get("check_cols"):
+            if self.check_cols:
                 raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'")
         # If the strategy is not 'check' or 'timestamp' it's a custom strategy,
         # formerly supported with GenericSnapshotConfig
 
-        if data.get("materialized") and data.get("materialized") != "snapshot":
+        if self.materialized and self.materialized != "snapshot":
             raise ValidationError("A snapshot must have a materialized value of 'snapshot'")
 
     # Called by "calculate_node_config_dict" in ContextConfigGenerator

core/dbt/cli/flags.py

@@ -289,6 +289,10 @@ def _assign_params(
             params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"]
         )
 
+        # Handle arguments mutually exclusive with INLINE
+        self._assert_mutually_exclusive(params_assigned_from_default, ["SELECT", "INLINE"])
+        self._assert_mutually_exclusive(params_assigned_from_default, ["SELECTOR", "INLINE"])
+
         # Support lower cased access for legacy code.
         params = set(
             x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__")
@@ -315,7 +319,9 @@ def _assert_mutually_exclusive(
         """
         set_flag = None
         for flag in group:
-            flag_set_by_user = flag.lower() not in params_assigned_from_default
+            flag_set_by_user = (
+                hasattr(self, flag) and flag.lower() not in params_assigned_from_default
+            )
             if flag_set_by_user and set_flag:
                 raise DbtUsageException(
                     f"{flag.lower()}: not allowed with argument {set_flag.lower()}"

core/dbt/parser/generic_test_builders.py

@@ -114,7 +114,8 @@ def __init__(
         self.package_name: str = package_name
         self.target: Testable = target
         self.version: Optional[NodeVersion] = version
-
+        self.render_ctx: Dict[str, Any] = render_ctx
+        self.column_name: Optional[str] = column_name
         self.args["model"] = self.build_model_str()
 
         match = self.TEST_NAME_PATTERN.match(test_name)
@@ -125,39 +126,12 @@ def __init__(
         self.name: str = groups["test_name"]
         self.namespace: str = groups["test_namespace"]
         self.config: Dict[str, Any] = {}
+        # Process legacy args
+        self.config.update(self._process_legacy_args())
 
-        # This code removes keys identified as config args from the test entry
-        # dictionary. The keys remaining in the 'args' dictionary will be
-        # "kwargs", or keyword args that are passed to the test macro.
-        # The "kwargs" are not rendered into strings until compilation time.
-        # The "configs" are rendered here (since they were not rendered back
-        # in the 'get_key_dicts' methods in the schema parsers).
-        for key in self.CONFIG_ARGS:
-            value = self.args.pop(key, None)
-            # 'modifier' config could be either top level arg or in config
-            if value and "config" in self.args and key in self.args["config"]:
-                raise SameKeyNestedError()
-            if not value and "config" in self.args:
-                value = self.args["config"].pop(key, None)
-            if isinstance(value, str):
-
-                try:
-                    value = get_rendered(value, render_ctx, native=True)
-                except UndefinedMacroError as e:
-
-                    raise CustomMacroPopulatingConfigValueError(
-                        target_name=self.target.name,
-                        column_name=column_name,
-                        name=self.name,
-                        key=key,
-                        err_msg=e.msg,
-                    )
-
-            if value is not None:
-                self.config[key] = value
-
+        # Process config args if present
         if "config" in self.args:
-            del self.args["config"]
+            self.config.update(self._render_values(self.args.pop("config", {})))
 
         if self.namespace is not None:
             self.package_name = self.namespace
@@ -182,6 +156,36 @@ def __init__(
             if short_name != full_name and "alias" not in self.config:
                 self.config["alias"] = short_name
 
+    def _process_legacy_args(self):
+        config = {}
+        for key in self.CONFIG_ARGS:
+            value = self.args.pop(key, None)
+            if value and "config" in self.args and key in self.args["config"]:
+                raise SameKeyNestedError()
+            if not value and "config" in self.args:
+                value = self.args["config"].pop(key, None)
+            config[key] = value
+
+        return self._render_values(config)
+
+    def _render_values(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        rendered_config = {}
+        for key, value in config.items():
+            if isinstance(value, str):
+                try:
+                    value = get_rendered(value, self.render_ctx, native=True)
+                except UndefinedMacroError as e:
+                    raise CustomMacroPopulatingConfigValueError(
+                        target_name=self.target.name,
+                        column_name=self.column_name,
+                        name=self.name,
+                        key=key,
+                        err_msg=e.msg,
+                    )
+            if value is not None:
+                rendered_config[key] = value
+        return rendered_config
+
     def _bad_type(self) -> TypeError:
         return TypeError('invalid target type "{}"'.format(type(self.target)))
 

core/dbt/parser/manifest.py

@@ -467,6 +467,7 @@ def load(self) -> Manifest:
             self.process_model_inferred_primary_keys()
             self.check_valid_group_config()
             self.check_valid_access_property()
+            self.check_valid_snapshot_config()
 
             semantic_manifest = SemanticManifest(self.manifest)
             if not semantic_manifest.validate():
@@ -1345,6 +1346,16 @@ def check_valid_access_property(self):
                     materialization=node.get_materialization(),
                 )
 
+    def check_valid_snapshot_config(self):
+        # Snapshot config can be set in either SQL files or yaml files,
+        # so we need to validate afterward.
+        for node in self.manifest.nodes.values():
+            if node.resource_type != NodeType.Snapshot:
+                continue
+            if node.created_at < self.started_at:
+                continue
+            node.config.final_validate()
+
     def write_perf_info(self, target_path: str):
         path = os.path.join(target_path, PERF_INFO_FILE_NAME)
         write_file(path, json.dumps(self._perf_info, cls=dbt.utils.JSONEncoder, indent=4))

core/dbt/task/compile.py

@@ -104,6 +104,12 @@ def _runtime_initialize(self):
                 )
                 sql_node = block_parser.parse_remote(self.args.inline, "inline_query")
                 process_node(self.config, self.manifest, sql_node)
+                # Special hack to remove disabled, if it's there. This would only happen
+                # if all models are disabled in dbt_project
+                if sql_node.config.enabled is False:
+                    sql_node.config.enabled = True
+                    self.manifest.disabled.pop(sql_node.unique_id)
+                    self.manifest.nodes[sql_node.unique_id] = sql_node
                 # keep track of the node added to the manifest
                 self._inline_node_id = sql_node.unique_id
             except CompilationError as exc:

core/dbt/task/printer.py

@@ -13,6 +13,7 @@
     StatsLine,
 )
 from dbt.node_types import NodeType
+from dbt_common.events.base_types import EventLevel
 from dbt_common.events.format import pluralize
 from dbt_common.events.functions import fire_event
 from dbt_common.events.types import Formatting
@@ -68,14 +69,13 @@ def print_run_status_line(results) -> None:
 
 
 def print_run_result_error(result, newline: bool = True, is_warning: bool = False) -> None:
-    if newline:
-        fire_event(Formatting(""))
-
     # set node_info for logging events
     node_info = None
     if hasattr(result, "node") and result.node:
         node_info = result.node.node_info
     if result.status == NodeStatus.Fail or (is_warning and result.status == NodeStatus.Warn):
+        if newline:
+            fire_event(Formatting(""))
         if is_warning:
             fire_event(
                 RunResultWarning(
@@ -112,8 +112,13 @@ def print_run_result_error(result, newline: bool = True, is_warning: bool = Fals
             fire_event(
                 CheckNodeTestFailure(relation_name=result.node.relation_name, node_info=node_info)
             )
-
+    elif result.status == NodeStatus.Skipped and result.message is not None:
+        if newline:
+            fire_event(Formatting(""), level=EventLevel.DEBUG)
+        fire_event(RunResultError(msg=result.message), level=EventLevel.DEBUG)
     elif result.message is not None:
+        if newline:
+            fire_event(Formatting(""))
         fire_event(RunResultError(msg=result.message, node_info=node_info))
 
 

core/dbt/task/runnable.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from multiprocessing.dummy import Pool as ThreadPool
 from pathlib import Path
-from typing import AbstractSet, Dict, Iterable, List, Optional, Set, Tuple
+from typing import AbstractSet, Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import dbt.exceptions
 import dbt.tracking
@@ -108,7 +108,11 @@ def exclusion_arg(self):
 
     def get_selection_spec(self) -> SelectionSpec:
         default_selector_name = self.config.get_default_selector_name()
-        if self.args.selector:
+        spec: Union[SelectionSpec, bool]
+        if hasattr(self.args, "inline") and self.args.inline:
+            # We want an empty selection spec.
+            spec = parse_difference(None, None)
+        elif self.args.selector:
             # use pre-defined selector (--selector)
             spec = self.config.get_selector(self.args.selector)
         elif not (self.selection_arg or self.exclusion_arg) and default_selector_name:

core/setup.py

@@ -69,7 +69,7 @@
         # Accept patches but avoid automatically updating past a set minor version range.
         "dbt-extractor>=0.5.0,<=0.6",
         "minimal-snowplow-tracker>=0.0.2,<0.1",
-        "dbt-semantic-interfaces>=0.5.1,<0.6",
+        "dbt-semantic-interfaces>=0.5.1,<0.7",
         # Minor versions for these are expected to be backwards-compatible
         "dbt-common>=1.3.0,<2.0",
         "dbt-adapters>=1.1.1,<2.0",

tests/functional/configs/test_configs.py

@@ -2,7 +2,6 @@
 
 import pytest
 
-from dbt.exceptions import ParsingError
 from dbt.tests.util import (
     check_relations_equal,
     run_dbt,
@@ -120,7 +119,7 @@ def test_snapshots_materialization_proj_config(self, project):
         snapshots_dir = os.path.join(project.project_root, "snapshots")
         write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql")
 
-        with pytest.raises(ParsingError):
+        with pytest.raises(ValidationError):
             run_dbt()
 
 

tests/functional/dbt_runner/test_dbt_runner.py

@@ -36,6 +36,9 @@ def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None:
         res = dbt.invoke(["deps", "--warn-error", "--warn-error-options", '{"include": "all"}'])
         assert type(res.exception) == DbtUsageException
 
+        res = dbt.invoke(["compile", "--select", "models", "--inline", "select 1 as id"])
+        assert type(res.exception) == DbtUsageException
+
     def test_invalid_command(self, dbt: dbtRunner) -> None:
         res = dbt.invoke(["invalid-command"])
         assert type(res.exception) == DbtUsageException

tests/functional/dependencies/test_dependency_options.py

@@ -39,7 +39,7 @@ def test_deps_lock(self, clean_start):
   - package: fivetran/fivetran_utils
     version: 0.4.7
   - package: dbt-labs/dbt_utils
-    version: 1.1.1
+    version: 1.2.0
 sha1_hash: 71304bca2138cf8004070b3573a1e17183c0c1a8
 """
         )
@@ -56,7 +56,7 @@ def test_deps_default(self, clean_start):
   - package: fivetran/fivetran_utils
     version: 0.4.7
   - package: dbt-labs/dbt_utils
-    version: 1.1.1
+    version: 1.2.0
 sha1_hash: 71304bca2138cf8004070b3573a1e17183c0c1a8
 """
         )