From 94bcaab55303c64ae6ed80f0a5a41cb5d1734764 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 8 Nov 2023 14:57:03 -0500 Subject: [PATCH] All commands support --defer --- core/dbt/cli/flags.py | 8 +- core/dbt/cli/main.py | 74 +++---------------- core/dbt/task/compile.py | 24 +----- core/dbt/task/freshness.py | 4 - core/dbt/task/list.py | 4 - core/dbt/task/runnable.py | 20 ++++- core/dbt/task/seed.py | 4 - .../defer_state/test_defer_state.py | 6 -- 8 files changed, 36 insertions(+), 108 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 2678d53b6dd..0055032e146 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -347,7 +347,13 @@ def add_fn(x): if k == "macro" and command == CliCommand.RUN_OPERATION: add_fn(v) # None is a Singleton, False is a Flyweight, only one instance of each. - elif v is None or v is False: + elif (v is None or v is False) and k not in ( + # These are None by default but they do not support --no-{flag} + "defer_state", + "warn_error", + "warn_error_options", + "log_format", + ): add_fn(f"--no-{spinal_cased}") elif v is True: add_fn(f"--{spinal_cased}") diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 2454a15a564..eadfef801bd 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -123,11 +123,18 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: def global_flags(func): @p.cache_selected_only @p.debug + @p.defer + @p.deprecated_defer + @p.defer_state + @p.deprecated_favor_state @p.deprecated_print + @p.deprecated_state @p.enable_legacy_logger @p.fail_fast + @p.favor_state @p.log_cache_events @p.log_file_max_bytes + @p.log_format @p.log_format_file @p.log_level @p.log_level_file @@ -143,12 +150,15 @@ def global_flags(func): @p.record_timing_info @p.send_anonymous_usage_stats @p.single_threaded + @p.state @p.static_parser @p.use_colors @p.use_colors_file @p.use_experimental_parser @p.version @p.version_check + @p.warn_error + @p.warn_error_options @p.write_json @functools.wraps(func) def wrapper(*args, **kwargs): @@ -166,9 +176,6 @@ def wrapper(*args, **kwargs): ) @click.pass_context @global_flags -@p.warn_error -@p.warn_error_options -@p.log_format @p.show_resource_report def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. @@ -180,11 +187,7 @@ def cli(ctx, **kwargs): @cli.command("build") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.full_refresh @p.include_saved_query @p.indirect_selection @@ -195,9 +198,6 @@ def cli(ctx, **kwargs): @p.select @p.selector @p.show -@p.state -@p.defer_state -@p.deprecated_state @p.store_failures @p.target @p.target_path @@ -259,11 +259,7 @@ def docs(ctx, **kwargs): @click.pass_context @global_flags @p.compile_docs -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.profile @p.profiles_dir @p.project_dir @@ -271,9 +267,6 @@ def docs(ctx, **kwargs): @p.selector @p.empty_catalog @p.static -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -330,11 +323,7 @@ def docs_serve(ctx, **kwargs): @cli.command("compile") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.full_refresh @p.show_output_format @p.indirect_selection @@ -346,9 +335,6 @@ def docs_serve(ctx, **kwargs): @p.select @p.selector @p.inline -@p.state -@p.defer_state -@p.deprecated_state @p.compile_inject_ephemeral_ctes @p.target @p.target_path @@ -378,11 +364,7 @@ def compile(ctx, **kwargs): @cli.command("show") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.full_refresh @p.show_output_format @p.show_limit @@ -394,9 +376,6 @@ def compile(ctx, **kwargs): @p.select @p.selector @p.inline -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -533,9 +512,6 @@ def init(ctx, **kwargs): @p.resource_type @p.raw_select @p.selector -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.vars @@ -591,10 +567,6 @@ def parse(ctx, **kwargs): @cli.command("run") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer -@p.favor_state -@p.deprecated_favor_state @p.exclude @p.full_refresh @p.profile @@ -603,9 +575,6 @@ def parse(ctx, **kwargs): @p.empty @p.select @p.selector -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -638,7 +607,6 @@ def run(ctx, **kwargs): @p.vars @p.profile @p.target -@p.state @p.threads @requires.postflight @requires.preflight @@ -663,7 +631,6 @@ def retry(ctx, **kwargs): @cli.command("clone") @click.pass_context @global_flags -@p.defer_state @p.exclude @p.full_refresh @p.profile @@ -672,7 +639,6 @@ def retry(ctx, **kwargs): @p.resource_type @p.select @p.selector -@p.state # required @p.target @p.target_path @p.threads @@ -740,9 +706,6 @@ def run_operation(ctx, **kwargs): @p.select @p.selector @p.show -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -769,19 +732,12 @@ def seed(ctx, **kwargs): @cli.command("snapshot") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.profile @p.profiles_dir @p.project_dir @p.select @p.selector -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -824,9 +780,6 @@ def source(ctx, **kwargs): @p.project_dir @p.select @p.selector -@p.state -@p.defer_state -@p.deprecated_state @p.target @p.target_path @p.threads @@ -860,20 +813,13 @@ def freshness(ctx, **kwargs): @cli.command("test") @click.pass_context @global_flags -@p.defer -@p.deprecated_defer @p.exclude -@p.favor_state -@p.deprecated_favor_state @p.indirect_selection @p.profile @p.profiles_dir @p.project_dir @p.select @p.selector -@p.state -@p.defer_state -@p.deprecated_state @p.store_failures @p.target @p.target_path diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index ba505e1a6ec..20db6ddc518 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -1,7 +1,5 @@ import threading -from typing import AbstractSet, Optional -from dbt.contracts.graph.manifest import WritableManifest from dbt.contracts.results import RunStatus, RunResult from dbt.events.base_types import EventLevel from dbt.events.functions import fire_event @@ -14,7 +12,7 @@ from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType -from dbt.parser.manifest import write_manifest, process_node +from dbt.parser.manifest import process_node from dbt.parser.sql import SqlBlockParser from dbt.task.base import BaseRunner from dbt.task.runnable import GraphRunnableTask @@ -101,26 +99,6 @@ def task_end_messages(self, results): ) ) - def _get_deferred_manifest(self) -> Optional[WritableManifest]: - return super()._get_deferred_manifest() if self.args.defer else None - - def defer_to_manifest(self, adapter, selected_uids: AbstractSet[str]): - deferred_manifest = self._get_deferred_manifest() - if deferred_manifest is None: - return - if self.manifest is None: - raise DbtInternalError( - "Expected to defer to manifest, but there is no runtime manifest to defer from!" - ) - self.manifest.merge_from_artifact( - adapter=adapter, - other=deferred_manifest, - selected=selected_uids, - favor_state=bool(self.args.favor_state), - ) - # TODO: is it wrong to write the manifest here? I think it's right... - write_manifest(self.manifest, self.config.project_target_path) - def _runtime_initialize(self): if getattr(self.args, "inline", None): try: diff --git a/core/dbt/task/freshness.py b/core/dbt/task/freshness.py index bb8a9c31ce3..eee15d905f3 100644 --- a/core/dbt/task/freshness.py +++ b/core/dbt/task/freshness.py @@ -171,10 +171,6 @@ def node_is_match(self, node): class FreshnessTask(GraphRunnableTask): - def defer_to_manifest(self, adapter, selected_uids): - # freshness don't defer - return - def result_path(self): if self.args.output: return os.path.realpath(self.args.output) diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 3b9448aeb9d..86fe3e926a1 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -186,10 +186,6 @@ def selection_arg(self): else: return self.args.select - def defer_to_manifest(self, adapter, selected_uids): - # list don't defer - return - def get_node_selector(self): if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 242e155a4d7..d61a4cb2e9f 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -130,9 +130,22 @@ def get_selection_spec(self) -> SelectionSpec: def get_node_selector(self) -> NodeSelector: raise NotImplementedError(f"get_node_selector not implemented for task {type(self)}") - @abstractmethod def defer_to_manifest(self, adapter, selected_uids: AbstractSet[str]): - raise NotImplementedError(f"defer_to_manifest not implemented for task {type(self)}") + deferred_manifest = self._get_deferred_manifest() + if deferred_manifest is None: + return + if self.manifest is None: + raise DbtInternalError( + "Expected to defer to manifest, but there is no runtime manifest to defer from!" + ) + self.manifest.merge_from_artifact( + adapter=adapter, + other=deferred_manifest, + selected=selected_uids, + favor_state=bool(self.args.favor_state), + ) + # TODO: is it wrong to write the manifest here? I think it's right... + write_manifest(self.manifest, self.config.project_target_path) def get_graph_queue(self) -> GraphQueue: selector = self.get_node_selector() @@ -605,6 +618,9 @@ def task_end_messages(self, results): print_run_end_messages(results) def _get_deferred_manifest(self) -> Optional[WritableManifest]: + if not self.args.defer: + return None + state = self.previous_defer_state or self.previous_state if not state: raise DbtRuntimeError( diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 9ec1df3b81f..82087e740dd 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -63,10 +63,6 @@ def print_result_line(self, result): class SeedTask(RunTask): - def defer_to_manifest(self, adapter, selected_uids): - # seeds don't defer - return - def raise_on_first_error(self): return False diff --git a/tests/functional/defer_state/test_defer_state.py b/tests/functional/defer_state/test_defer_state.py index 3a139f8aa47..102345fdf6e 100644 --- a/tests/functional/defer_state/test_defer_state.py +++ b/tests/functional/defer_state/test_defer_state.py @@ -5,7 +5,6 @@ import pytest -from dbt.cli.exceptions import DbtUsageException from dbt.contracts.results import RunStatus from dbt.exceptions import DbtRuntimeError from dbt.tests.util import run_dbt, write_file, rm_file @@ -105,11 +104,6 @@ def run_and_save_state(self, project_root, with_snapshot=False): class TestDeferStateUnsupportedCommands(BaseDeferState): - def test_unsupported_commands(self, project): - # make sure these commands don"t work with --defer - with pytest.raises(DbtUsageException): - run_dbt(["seed", "--defer"]) - def test_no_state(self, project): # no "state" files present, snapshot fails with pytest.raises(DbtRuntimeError):