Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provide ability to exclude resource_types, instead of listing everything not excluded #9756

Merged
merged 11 commits into from
Mar 13, 2024
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240312-140407.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Allow excluding resource types for build, list, and clone commands
time: 2024-03-12T14:04:07.086017-04:00
custom:
Author: gshank
Issue: "9237"
5 changes: 4 additions & 1 deletion core/dbt/cli/flags.py
Original file line number Diff line number Diff line change
@@ -399,7 +399,10 @@ def add_fn(x):

# MultiOption flags come back as lists, but we want to pass them as space separated strings
if isinstance(v, list):
v = " ".join(v)
if len(v) > 0:
v = " ".join(v)
else:
continue

if k == "macro" and command == CliCommand.RUN_OPERATION:
add_fn(v)
3 changes: 3 additions & 0 deletions core/dbt/cli/main.py
Original file line number Diff line number Diff line change
@@ -194,6 +194,7 @@ def cli(ctx, **kwargs):
@p.profiles_dir
@p.project_dir
@p.resource_type
@p.exclude_resource_type
@p.select
@p.selector
@p.show
@@ -499,6 +500,7 @@ def init(ctx, **kwargs):
@p.profiles_dir
@p.project_dir
@p.resource_type
@p.exclude_resource_type
@p.raw_select
@p.selector
@p.target
@@ -627,6 +629,7 @@ def retry(ctx, **kwargs):
@p.profiles_dir
@p.project_dir
@p.resource_type
@p.exclude_resource_type
@p.select
@p.selector
@p.target
30 changes: 29 additions & 1 deletion core/dbt/cli/params.py
Original file line number Diff line number Diff line change
@@ -391,7 +391,7 @@
resource_type = click.option(
"--resource-types",
"--resource-type",
envvar=None,
envvar="DBT_RESOURCE_TYPES",
help="Restricts the types of resources that dbt will include",
type=ChoiceTuple(
[
@@ -402,6 +402,7 @@
"analysis",
"model",
"test",
"unit_test",
"exposure",
"snapshot",
"seed",
@@ -415,6 +416,33 @@
default=(),
)

exclude_resource_type = click.option(
"--exclude-resource-types",
"--exclude-resource-type",
envvar="DBT_EXCLUDE_RESOURCE_TYPES",
help="Specify the types of resources that dbt will exclude",
type=ChoiceTuple(
[
"metric",
"semantic_model",
"saved_query",
"source",
"analysis",
"model",
"test",
"unit_test",
"exposure",
"snapshot",
"seed",
"default",
],
case_sensitive=False,
),
cls=MultiOption,
multiple=True,
default=(),
)

# Renamed to --export-saved-queries
deprecated_include_saved_query = click.option(
"--include-saved-query/--no-include-saved-query",
30 changes: 29 additions & 1 deletion core/dbt/task/base.py
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@
from contextlib import nullcontext
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, Union
from typing import Any, Dict, List, Optional, Type, Union, Set

from dbt.compilation import Compiler
import dbt_common.exceptions.base
@@ -16,6 +16,7 @@
from dbt.config.profile import read_profile
from dbt.constants import DBT_PROJECT_FILE_NAME
from dbt.contracts.graph.manifest import Manifest
from dbt.artifacts.resources.types import NodeType
from dbt.artifacts.schemas.results import TimingInfo, collect_timing_info
from dbt.artifacts.schemas.results import NodeStatus, RunningStatus, RunStatus
from dbt.artifacts.schemas.run import RunResult
@@ -480,3 +481,30 @@
def do_skip(self, cause=None):
self.skip = True
self.skip_cause = cause


def resource_types_from_args(
args, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType]
) -> Set[NodeType]:

if not args.resource_types:
resource_types = default_resource_values
else:
# This is a list of strings, not NodeTypes
arg_resource_types = set(args.resource_types)

if "all" in arg_resource_types:
arg_resource_types.remove("all")
arg_resource_types.update(all_resource_values)
if "default" in arg_resource_types:
arg_resource_types.remove("default")
arg_resource_types.update(default_resource_values)
# Convert to a set of NodeTypes now that the non-NodeType strings are gone
resource_types = set([NodeType(rt) for rt in arg_resource_types])

if args.exclude_resource_types:
# Convert from a list of strings to a set of NodeTypes
exclude_resource_types = set([NodeType(rt) for rt in args.exclude_resource_types])
resource_types = resource_types - exclude_resource_types

Check warning on line 508 in core/dbt/task/base.py

Codecov / codecov/patch

core/dbt/task/base.py#L507-L508

Added lines #L507 - L508 were not covered by tests

return resource_types
13 changes: 4 additions & 9 deletions core/dbt/task/build.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
from dbt.graph import ResourceTypeSelector, GraphQueue, Graph
from dbt.node_types import NodeType
from dbt.task.test import TestSelector
from dbt.task.base import BaseRunner
from dbt.task.base import BaseRunner, resource_types_from_args
from dbt_common.events.functions import fire_event
from dbt.events.types import LogNodeNoOpResult
from dbt.exceptions import DbtInternalError
@@ -80,14 +80,9 @@ def __init__(self, args, config, manifest) -> None:
self.model_to_unit_test_map: Dict[str, List] = {}

def resource_types(self, no_unit_tests=False):
if not self.args.resource_types:
resource_types = list(self.ALL_RESOURCE_VALUES)
else:
resource_types = set(self.args.resource_types)

if "all" in resource_types:
resource_types.remove("all")
resource_types.update(self.ALL_RESOURCE_VALUES)
resource_types = resource_types_from_args(
self.args, set(self.ALL_RESOURCE_VALUES), set(self.ALL_RESOURCE_VALUES)
)

# First we get selected_nodes including unit tests, then without,
# and do a set difference.
21 changes: 8 additions & 13 deletions core/dbt/task/clone.py
Original file line number Diff line number Diff line change
@@ -9,8 +9,8 @@
from dbt_common.dataclass_schema import dbtClassMixin
from dbt_common.exceptions import DbtInternalError, CompilationError
from dbt.graph import ResourceTypeSelector
from dbt.node_types import NodeType, REFABLE_NODE_TYPES
from dbt.task.base import BaseRunner
from dbt.node_types import REFABLE_NODE_TYPES
from dbt.task.base import BaseRunner, resource_types_from_args
from dbt.task.run import _validate_materialization_relations_dict
from dbt.task.runnable import GraphRunnableTask

@@ -132,18 +132,13 @@ def before_run(self, adapter, selected_uids: AbstractSet[str]):

@property
def resource_types(self):
if not self.args.resource_types:
return REFABLE_NODE_TYPES

values = set(self.args.resource_types)

if "all" in values:
values.remove("all")
values.update(REFABLE_NODE_TYPES)

values = [NodeType(val) for val in values if val in REFABLE_NODE_TYPES]
resource_types = resource_types_from_args(
self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES)
)

return list(values)
# filter out any non-refable node types
resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES]
return list(resource_types)

def get_node_selector(self) -> ResourceTypeSelector:
resource_types = self.resource_types
17 changes: 6 additions & 11 deletions core/dbt/task/list.py
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
)
from dbt.flags import get_flags
from dbt.graph import ResourceTypeSelector
from dbt.task.base import resource_types_from_args
from dbt.task.runnable import GraphRunnableTask
from dbt.task.test import TestSelector
from dbt.node_types import NodeType
@@ -183,17 +184,11 @@ def resource_types(self):
if self.args.models:
return [NodeType.Model]

if not self.args.resource_types:
return list(self.DEFAULT_RESOURCE_VALUES)

values = set(self.args.resource_types)
if "default" in values:
values.remove("default")
values.update(self.DEFAULT_RESOURCE_VALUES)
if "all" in values:
values.remove("all")
values.update(self.ALL_RESOURCE_VALUES)
return list(values)
resource_types = resource_types_from_args(
self.args, set(self.ALL_RESOURCE_VALUES), set(self.DEFAULT_RESOURCE_VALUES)
)

return list(resource_types)

@property
def selection_arg(self):
12 changes: 11 additions & 1 deletion tests/functional/unit_testing/test_unit_testing.py
Original file line number Diff line number Diff line change
@@ -50,12 +50,22 @@ def test_basic(self, project):
results = run_dbt(["test", "--select", "my_model"], expect_pass=False)
assert len(results) == 5

results = run_dbt(["build", "--select", "my_model"], expect_pass=False)
results = run_dbt(
["build", "--select", "my_model", "--resource-types", "model unit_test"],
expect_pass=False,
)
assert len(results) == 6
for result in results:
if result.node.unique_id == "model.test.my_model":
result.status == NodeStatus.Skipped

# Run build command but specify no unit tests
results = run_dbt(
["build", "--select", "my_model", "--exclude-resource-types", "unit_test"],
expect_pass=True,
)
assert len(results) == 1

# Test select by test name
results = run_dbt(["test", "--select", "test_name:test_my_model_string_concat"])
assert len(results) == 1