Merge remote-tracking branch 'origin/main' into paw/test-fixture-env

.changes/unreleased/Dependencies-20240117-100818.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Relax pathspec upper bound version restriction
+time: 2024-01-17T10:08:18.009949641+01:00
+custom:
+  Author: rzjfr
+  PR: "9373"

.changes/unreleased/Dependencies-20240331-103917.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Remove duplicate dependency of protobuf in dev-requirements
+time: 2024-03-31T10:39:17.432017-07:00
+custom:
+  Author: niteshy
+  Issue: "9830"

.changes/unreleased/Features-20240405-175733.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Better error message when trying to select a disabled model
+time: 2024-04-05T17:57:33.047963+02:00
+custom:
+  Author: SamuelBFavarin
+  Issue: "9747"

.changes/unreleased/Fixes-20240108-232035.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: fix configuration of turning test warnings into failures with WARN_ERROR_OPTIONS
+time: 2024-01-08T23:20:35.339102+09:00
+custom:
+  Author: jx2lee
+  Issue: "7761"

.changes/unreleased/Fixes-20240323-124558.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Only create the packages-install-path / dbt_packages folder during dbt deps
+time: 2024-03-23T12:45:58.159017-06:00
+custom:
+  Author: dbeatty10
+  Issue: 6985 9584

.changes/unreleased/Fixes-20240402-135556.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Exclude password-like fields for considering reparse
+time: 2024-04-02T13:55:56.169953-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "9795"

core/dbt/cli/flags.py

@@ -352,6 +352,11 @@ def set_common_global_flags(self):
         if getattr(self, "MACRO_DEBUGGING", None) is not None:
             jinja.MACRO_DEBUGGING = getattr(self, "MACRO_DEBUGGING")
 
+    # This is here to prevent mypy from complaining about all of the
+    # attributes which we added dynamically.
+    def __getattr__(self, name: str) -> Any:
+        return super().__get_attribute__(name)  # type: ignore
+
 
 CommandParams = List[str]
 

core/dbt/cli/main.py

@@ -399,7 +399,6 @@ def debug(ctx, **kwargs):
 
     task = DebugTask(
         ctx.obj["flags"],
-        None,
     )
 
     results = task.run()
@@ -464,7 +463,7 @@ def init(ctx, **kwargs):
     """Initialize a new dbt project."""
     from dbt.task.init import InitTask
 
-    task = InitTask(ctx.obj["flags"], None)
+    task = InitTask(ctx.obj["flags"])
 
     results = task.run()
     success = task.interpret_results(results)

core/dbt/compilation.py

@@ -274,7 +274,6 @@ def __init__(self, config) -> None:
 
     def initialize(self):
         make_directory(self.config.project_target_path)
-        make_directory(self.config.packages_install_path)
 
     # creates a ModelContext which is converted to
     # a dict for jinja rendering of SQL

core/dbt/deps/resolver.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import Dict, List, NoReturn, Union, Type, Iterator, Set, Any
+from typing import Dict, List, NoReturn, Type, Iterator, Set, Any
 
 from dbt.exceptions import (
     DuplicateDependencyToRootError,
@@ -17,14 +17,13 @@
 from dbt.deps.registry import RegistryUnpinnedPackage
 
 from dbt.contracts.project import (
+    PackageSpec,
     LocalPackage,
     TarballPackage,
     GitPackage,
     RegistryPackage,
 )
 
-PackageContract = Union[LocalPackage, TarballPackage, GitPackage, RegistryPackage]
-
 
 @dataclass
 class PackageListing:
@@ -68,7 +67,7 @@ def incorporate(self, package: UnpinnedPackage):
         else:
             self.packages[key] = package
 
-    def update_from(self, src: List[PackageContract]) -> None:
+    def update_from(self, src: List[PackageSpec]) -> None:
         pkg: UnpinnedPackage
         for contract in src:
             if isinstance(contract, LocalPackage):
@@ -84,9 +83,7 @@ def update_from(self, src: List[PackageContract]) -> None:
             self.incorporate(pkg)
 
     @classmethod
-    def from_contracts(
-        cls: Type["PackageListing"], src: List[PackageContract]
-    ) -> "PackageListing":
+    def from_contracts(cls: Type["PackageListing"], src: List[PackageSpec]) -> "PackageListing":
         self = cls({})
         self.update_from(src)
         return self
@@ -114,7 +111,7 @@ def _check_for_duplicate_project_names(
 
 
 def resolve_packages(
-    packages: List[PackageContract],
+    packages: List[PackageSpec],
     project: Project,
     cli_vars: Dict[str, Any],
 ) -> List[PinnedPackage]:
@@ -137,7 +134,7 @@ def resolve_packages(
     return resolved
 
 
-def resolve_lock_packages(packages: List[PackageContract]) -> List[PinnedPackage]:
+def resolve_lock_packages(packages: List[PackageSpec]) -> List[PinnedPackage]:
     lock_packages = PackageListing.from_contracts(packages)
     final = PackageListing()
 

core/dbt/events/types.py

@@ -1099,7 +1099,7 @@ def code(self) -> str:
         return "M030"
 
     def message(self) -> str:
-        return f"The selection criterion '{self.spec_raw}' does not match any nodes"
+        return f"The selection criterion '{self.spec_raw}' does not match any enabled nodes"
 
 
 class DepsLockUpdating(InfoLevel):

core/dbt/parser/manifest.py

@@ -835,13 +835,6 @@ def is_partial_parsable(self, manifest: Manifest) -> Tuple[bool, Optional[str]]:
             )
             valid = False
             reparse_reason = ReparseReason.proj_env_vars_changed
-        if (
-            self.manifest.state_check.profile_env_vars_hash
-            != manifest.state_check.profile_env_vars_hash
-        ):
-            fire_event(UnableToPartialParse(reason="env vars used in profiles.yml have changed"))
-            valid = False
-            reparse_reason = ReparseReason.prof_env_vars_changed
 
         missing_keys = {
             k
@@ -980,18 +973,18 @@ def build_manifest_state_check(self):
             env_var_str += f"{key}:{config.project_env_vars[key]}|"
         project_env_vars_hash = FileHash.from_contents(env_var_str)
 
-        # Create a FileHash of the env_vars in the project
-        key_list = list(config.profile_env_vars.keys())
-        key_list.sort()
-        env_var_str = ""
-        for key in key_list:
-            env_var_str += f"{key}:{config.profile_env_vars[key]}|"
-        profile_env_vars_hash = FileHash.from_contents(env_var_str)
+        # Create a hash of the connection_info, which user has access to in
+        # jinja context. Thus attributes here may affect the parsing result.
+        # Ideally we should not expose all of the connection info to the jinja.
 
-        # Create a FileHash of the profile file
-        profile_path = os.path.join(get_flags().PROFILES_DIR, "profiles.yml")
-        with open(profile_path) as fp:
-            profile_hash = FileHash.from_contents(fp.read())
+        # Renaming this variable mean that we will have to do a whole lot more
+        # change to make sure the previous manifest can be loaded correctly.
+        # This is an example of naming should be chosen based on the functionality
+        # rather than the implementation details.
+        connection_keys = list(config.credentials.connection_info())
+        # avoid reparsing because of ordering issues
+        connection_keys.sort()
+        profile_hash = FileHash.from_contents(pprint.pformat(connection_keys))
 
         # Create a FileHashes for dbt_project for all dependencies
         project_hashes = {}
@@ -1003,7 +996,6 @@ def build_manifest_state_check(self):
         # Create the ManifestStateCheck object
         state_check = ManifestStateCheck(
             project_env_vars_hash=project_env_vars_hash,
-            profile_env_vars_hash=profile_env_vars_hash,
             vars_hash=vars_hash,
             profile_hash=profile_hash,
             project_hashes=project_hashes,

core/dbt/task/base.py

@@ -6,13 +6,14 @@
 from contextlib import nullcontext
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Type, Union, Set
+from typing import Any, Dict, List, Optional, Set
 
 from dbt.compilation import Compiler
 import dbt_common.exceptions.base
 import dbt.exceptions
 from dbt import tracking
-from dbt.config import RuntimeConfig, Project
+from dbt.cli.flags import Flags
+from dbt.config import RuntimeConfig
 from dbt.config.profile import read_profile
 from dbt.constants import DBT_PROJECT_FILE_NAME
 from dbt.contracts.graph.manifest import Manifest
@@ -49,12 +50,6 @@
 from dbt.task.printer import print_run_result_error
 
 
-class NoneConfig:
-    @classmethod
-    def from_args(cls, args):
-        return None
-
-
 def read_profiles(profiles_dir=None):
     """This is only used for some error handling"""
     if profiles_dir is None:
@@ -71,15 +66,11 @@ def read_profiles(profiles_dir=None):
 
 
 class BaseTask(metaclass=ABCMeta):
-    ConfigType: Union[Type[NoneConfig], Type[Project]] = NoneConfig
-
-    def __init__(self, args, config, project=None) -> None:
+    def __init__(self, args: Flags) -> None:
         self.args = args
-        self.config = config
-        self.project = config if isinstance(config, Project) else project
 
     @classmethod
-    def pre_init_hook(cls, args):
+    def pre_init_hook(cls, args: Flags):
         """A hook called before the task is initialized."""
         if args.log_format == "json":
             log_manager.format_json()
@@ -93,23 +84,6 @@ def set_log_format(cls):
         else:
             log_manager.format_text()
 
-    @classmethod
-    def from_args(cls, args, *pargs, **kwargs):
-        try:
-            # This is usually RuntimeConfig
-            config = cls.ConfigType.from_args(args)
-        except dbt.exceptions.DbtProjectError as exc:
-            fire_event(LogDbtProjectError(exc=str(exc)))
-
-            tracking.track_invalid_invocation(args=args, result_type=exc.result_type)
-            raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc
-        except dbt.exceptions.DbtProfileError as exc:
-            all_profile_names = list(read_profiles(get_flags().PROFILES_DIR).keys())
-            fire_event(LogDbtProfileError(exc=str(exc), profiles=all_profile_names))
-            tracking.track_invalid_invocation(args=args, result_type=exc.result_type)
-            raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc
-        return cls(args, config, *pargs, **kwargs)
-
     @abstractmethod
     def run(self):
         raise dbt_common.exceptions.base.NotImplementedError("Not Implemented")
@@ -153,10 +127,11 @@ def move_to_nearest_project_dir(project_dir: Optional[str]) -> Path:
 # produce the same behavior. currently this class only contains manifest compilation,
 # holding a manifest, and moving direcories.
 class ConfiguredTask(BaseTask):
-    ConfigType = RuntimeConfig
-
-    def __init__(self, args, config, manifest: Optional[Manifest] = None) -> None:
-        super().__init__(args, config)
+    def __init__(
+        self, args: Flags, config: RuntimeConfig, manifest: Optional[Manifest] = None
+    ) -> None:
+        super().__init__(args)
+        self.config = config
         self.graph: Optional[Graph] = None
         self.manifest = manifest
         self.compiler = Compiler(self.config)
@@ -174,9 +149,22 @@ def compile_manifest(self):
             dbt.tracking.track_runnable_timing({"graph_compilation_elapsed": compile_time})
 
     @classmethod
-    def from_args(cls, args, *pargs, **kwargs):
+    def from_args(cls, args: Flags, *pargs, **kwargs):
         move_to_nearest_project_dir(args.project_dir)
-        return super().from_args(args, *pargs, **kwargs)
+        try:
+            # This is usually RuntimeConfig
+            config = RuntimeConfig.from_args(args)
+        except dbt.exceptions.DbtProjectError as exc:
+            fire_event(LogDbtProjectError(exc=str(exc)))
+
+            tracking.track_invalid_invocation(args=args, result_type=exc.result_type)
+            raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc
+        except dbt.exceptions.DbtProfileError as exc:
+            all_profile_names = list(read_profiles(get_flags().PROFILES_DIR).keys())
+            fire_event(LogDbtProfileError(exc=str(exc), profiles=all_profile_names))
+            tracking.track_invalid_invocation(args=args, result_type=exc.result_type)
+            raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc
+        return cls(args, config, *pargs, **kwargs)
 
 
 class ExecutionContext:
@@ -487,7 +475,7 @@ def do_skip(self, cause=None):
 
 
 def resource_types_from_args(
-    args, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType]
+    args: Flags, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType]
 ) -> Set[NodeType]:
 
     if not args.resource_types:

core/dbt/task/build.py

@@ -8,6 +8,9 @@
 
 from dbt.artifacts.schemas.results import NodeStatus, RunStatus
 from dbt.artifacts.schemas.run import RunResult
+from dbt.cli.flags import Flags
+from dbt.config.runtime import RuntimeConfig
+from dbt.contracts.graph.manifest import Manifest
 from dbt.graph import ResourceTypeSelector, GraphQueue, Graph
 from dbt.node_types import NodeType
 from dbt.task.test import TestSelector
@@ -74,7 +77,7 @@ class BuildTask(RunTask):
     }
     ALL_RESOURCE_VALUES = frozenset({x for x in RUNNER_MAP.keys()})
 
-    def __init__(self, args, config, manifest) -> None:
+    def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None:
         super().__init__(args, config, manifest)
         self.selected_unit_tests: Set = set()
         self.model_to_unit_test_map: Dict[str, List] = {}

core/dbt/task/clean.py

@@ -9,13 +9,20 @@
     FinishedCleanPaths,
 )
 from dbt_common.exceptions import DbtRuntimeError
+from dbt.cli.flags import Flags
+from dbt.config.project import Project
 from dbt.task.base import (
     BaseTask,
     move_to_nearest_project_dir,
 )
 
 
 class CleanTask(BaseTask):
+    def __init__(self, args: Flags, config: Project):
+        super().__init__(args)
+        self.config = config
+        self.project = config
+
     def run(self):
         """
         This function takes all the paths in the target file

core/dbt/task/debug.py

@@ -19,6 +19,7 @@
 import dbt.exceptions
 import dbt_common.exceptions
 from dbt.adapters.factory import get_adapter, register_adapter
+from dbt.cli.flags import Flags
 from dbt.config import PartialProject, Project, Profile
 from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
 from dbt.artifacts.schemas.results import RunStatus
@@ -77,8 +78,8 @@ class DebugRunStatus(Flag):
 
 
 class DebugTask(BaseTask):
-    def __init__(self, args, config) -> None:
-        super().__init__(args, config)
+    def __init__(self, args: Flags) -> None:
+        super().__init__(args)
         self.profiles_dir = args.PROFILES_DIR
         self.profile_path = os.path.join(self.profiles_dir, "profiles.yml")
         try:
@@ -97,13 +98,6 @@ def __init__(self, args, config) -> None:
         self.profile: Optional[Profile] = None
         self.raw_profile_data: Optional[Dict[str, Any]] = None
         self.profile_name: Optional[str] = None
-        self.project: Optional[Project] = None
-
-    @property
-    def project_profile(self):
-        if self.project is None:
-            return None
-        return self.project.profile_name
 
     def run(self) -> bool:
         # WARN: this is a legacy workflow that is not compatible with other runtime flags