From 60315b6a750c2065db342ec26ba8c1696325f7e6 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 7 Aug 2024 12:38:13 -0400 Subject: [PATCH] backport: buffer deprecations to respect --quiet and --warn-error-options (#10541) --- .../unreleased/Fixes-20240806-194843.yaml | 6 +++ core/dbt/cli/flags.py | 7 ++- core/dbt/config/project.py | 4 +- core/dbt/deprecations.py | 27 ++++++----- .../deprecations/test_deprecations.py | 48 +++++++++++++++++-- tests/unit/test_deprecations.py | 43 ++++++++++++----- tests/unit/test_internal_deprecations.py | 15 ++++++ 7 files changed, 120 insertions(+), 30 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240806-194843.yaml create mode 100644 tests/unit/test_internal_deprecations.py diff --git a/.changes/unreleased/Fixes-20240806-194843.yaml b/.changes/unreleased/Fixes-20240806-194843.yaml new file mode 100644 index 00000000000..7eb5a4bd8d8 --- /dev/null +++ b/.changes/unreleased/Fixes-20240806-194843.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: respect --quiet and --warn-error-options for flag deprecations +time: 2024-08-06T19:48:43.399453-04:00 +custom: + Author: michelleark + Issue: "10105" diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 1deaa0e7073..73471e41d5b 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -13,13 +13,14 @@ from dbt.cli.types import Command as CliCommand from dbt.config.project import read_project_flags from dbt.contracts.project import ProjectFlags +from dbt.deprecations import fire_buffered_deprecations, renamed_env_var +from dbt.events import ALL_EVENT_NAMES from dbt_common import ui from dbt_common.events import functions from dbt_common.exceptions import DbtInternalError from dbt_common.clients import jinja -from dbt.deprecations import renamed_env_var from dbt_common.helper_types import WarnErrorOptions -from dbt.events import ALL_EVENT_NAMES + if os.name != "nt": # https://bugs.python.org/issue41567 @@ -347,6 +348,8 @@ def fire_deprecations(self): # not get pickled when written to disk as json. object.__delattr__(self, "deprecated_env_var_warnings") + fire_buffered_deprecations() + @classmethod def from_dict(cls, command: CliCommand, args_dict: Dict[str, Any]) -> "Flags": command_arg_list = command_params(command, args_dict) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index deaa99df343..739dd2d6737 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -832,8 +832,8 @@ def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags: if profile_project_flags: # This can't use WARN_ERROR or WARN_ERROR_OPTIONS because they're in - # the config that we're loading. Uses special "warn" method. - deprecations.warn("project-flags-moved") + # the config that we're loading. Uses special "buffer" method and fired after flags are initialized in preflight. + deprecations.buffer("project-flags-moved") project_flags = profile_project_flags if project_flags is not None: diff --git a/core/dbt/deprecations.py b/core/dbt/deprecations.py index 86c820b6553..87635afb3e7 100644 --- a/core/dbt/deprecations.py +++ b/core/dbt/deprecations.py @@ -1,10 +1,10 @@ import abc -from typing import Optional, Set, List, Dict, ClassVar +from typing import Callable, ClassVar, Dict, List, Optional, Set import dbt.tracking from dbt.events import types as core_types -from dbt_common.events.functions import warn_or_error, fire_event +from dbt_common.events.functions import warn_or_error class DBTDeprecation: @@ -108,15 +108,6 @@ class ProjectFlagsMovedDeprecation(DBTDeprecation): _name = "project-flags-moved" _event = "ProjectFlagsMovedDeprecation" - def show(self, *args, **kwargs) -> None: - if self.name not in active_deprecations: - event = self.event(**kwargs) - # We can't do warn_or_error because the ProjectFlags - # is where that is set up and we're just reading it. - fire_event(event) - self.track_deprecation_warn() - active_deprecations.add(self.name) - class PackageMaterializationOverrideDeprecation(DBTDeprecation): _name = "package-materialization-override" @@ -156,6 +147,13 @@ def warn(name, *args, **kwargs): deprecations[name].show(*args, **kwargs) +def buffer(name: str, *args, **kwargs): + def show_callback(): + deprecations[name].show(*args, **kwargs) + + buffered_deprecations.append(show_callback) + + # these are globally available # since modules are only imported once, active_deprecations is a singleton @@ -179,6 +177,13 @@ def warn(name, *args, **kwargs): deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list} +buffered_deprecations: List[Callable] = [] + def reset_deprecations(): active_deprecations.clear() + + +def fire_buffered_deprecations(): + [dep_fn() for dep_fn in buffered_deprecations] + buffered_deprecations.clear() diff --git a/tests/functional/deprecations/test_deprecations.py b/tests/functional/deprecations/test_deprecations.py index d93cb5f9a10..b39347fffba 100644 --- a/tests/functional/deprecations/test_deprecations.py +++ b/tests/functional/deprecations/test_deprecations.py @@ -2,11 +2,12 @@ import dbt_common from dbt import deprecations +from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file +from dbt_common.exceptions import EventCompilationError from tests.functional.deprecations.fixtures import ( models_trivial__model_sql, bad_name_yaml, ) -from dbt.tests.util import run_dbt, write_file import yaml @@ -143,6 +144,45 @@ def models(self): def test_profile_config_deprecation(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == set() - run_dbt(["parse"]) - expected = {"project-flags-moved"} - assert expected == deprecations.active_deprecations + + _, logs = run_dbt_and_capture(["parse"]) + + assert ( + "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." + in logs + ) + assert deprecations.active_deprecations == {"project-flags-moved"} + + +class TestProjectFlagsMovedDeprecationQuiet(TestProjectFlagsMovedDeprecation): + def test_profile_config_deprecation(self, project): + deprecations.reset_deprecations() + assert deprecations.active_deprecations == set() + + _, logs = run_dbt_and_capture(["--quiet", "parse"]) + + assert ( + "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." + not in logs + ) + assert deprecations.active_deprecations == {"project-flags-moved"} + + +class TestProjectFlagsMovedDeprecationWarnErrorOptions(TestProjectFlagsMovedDeprecation): + def test_profile_config_deprecation(self, project): + deprecations.reset_deprecations() + with pytest.raises(EventCompilationError): + run_dbt(["--warn-error-options", "{'include': 'all'}", "parse"]) + + with pytest.raises(EventCompilationError): + run_dbt( + ["--warn-error-options", "{'include': ['ProjectFlagsMovedDeprecation']}", "parse"] + ) + + _, logs = run_dbt_and_capture( + ["--warn-error-options", "{'silence': ['ProjectFlagsMovedDeprecation']}", "parse"] + ) + assert ( + "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." + not in logs + ) diff --git a/tests/unit/test_deprecations.py b/tests/unit/test_deprecations.py index ca8b8006cbc..3b773cce070 100644 --- a/tests/unit/test_deprecations.py +++ b/tests/unit/test_deprecations.py @@ -1,15 +1,36 @@ -from dbt.internal_deprecations import deprecated -from dbt.flags import set_from_args -from argparse import Namespace +import pytest +import dbt.deprecations as deprecations -@deprecated(reason="just because", version="1.23.0", suggested_action="Make some updates") -def to_be_decorated(): - return 5 +@pytest.fixture(scope="function") +def active_deprecations(): + assert not deprecations.active_deprecations -# simple test that the return value is not modified -def test_deprecated_func(): - set_from_args(Namespace(WARN_ERROR=False), None) - assert hasattr(to_be_decorated, "__wrapped__") - assert to_be_decorated() == 5 + yield deprecations.active_deprecations + + deprecations.reset_deprecations() + + +@pytest.fixture(scope="function") +def buffered_deprecations(): + assert not deprecations.buffered_deprecations + + yield deprecations.buffered_deprecations + + deprecations.buffered_deprecations.clear() + + +def test_buffer_deprecation(active_deprecations, buffered_deprecations): + deprecations.buffer("project-flags-moved") + + assert active_deprecations == set() + assert len(buffered_deprecations) == 1 + + +def test_fire_buffered_deprecations(active_deprecations, buffered_deprecations): + deprecations.buffer("project-flags-moved") + deprecations.fire_buffered_deprecations() + + assert active_deprecations == set(["project-flags-moved"]) + assert len(buffered_deprecations) == 0 diff --git a/tests/unit/test_internal_deprecations.py b/tests/unit/test_internal_deprecations.py new file mode 100644 index 00000000000..ca8b8006cbc --- /dev/null +++ b/tests/unit/test_internal_deprecations.py @@ -0,0 +1,15 @@ +from dbt.internal_deprecations import deprecated +from dbt.flags import set_from_args +from argparse import Namespace + + +@deprecated(reason="just because", version="1.23.0", suggested_action="Make some updates") +def to_be_decorated(): + return 5 + + +# simple test that the return value is not modified +def test_deprecated_func(): + set_from_args(Namespace(WARN_ERROR=False), None) + assert hasattr(to_be_decorated, "__wrapped__") + assert to_be_decorated() == 5