From c63ae89efbb10a1974f9f9c04817125b168c1e75 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 16 Apr 2024 07:31:54 -0500 Subject: [PATCH 01/17] bump sqlparse to 0.5 (#9951) * bump sqlparse * changelog --- .changes/unreleased/Dependencies-20240415-202426.yaml | 6 ++++++ core/setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240415-202426.yaml diff --git a/.changes/unreleased/Dependencies-20240415-202426.yaml b/.changes/unreleased/Dependencies-20240415-202426.yaml new file mode 100644 index 00000000000..d009f63a154 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240415-202426.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Bump sqlparse to >=0.5.0, <0.6.0 +time: 2024-04-15T20:24:26.768707-05:00 +custom: + Author: emmyoop + Issue: "9951" diff --git a/core/setup.py b/core/setup.py index 7cbbfb7e41e..83b150b1f74 100644 --- a/core/setup.py +++ b/core/setup.py @@ -67,7 +67,7 @@ # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) # and check compatibility / bump in each new minor version of dbt-core. "pathspec>=0.9,<0.13", - "sqlparse>=0.2.3,<0.5", + "sqlparse>=0.5.0,<0.6.0", # ---- # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. From 11dbe679b93b8e1e992c349f556b3728ccaff443 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 16 Apr 2024 07:32:09 -0500 Subject: [PATCH 02/17] stop bumping version in DockerFile (#9950) --- .bumpversion.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bfc93ed7dcf..6e036340dd6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -35,5 +35,3 @@ first_value = 1 [bumpversion:file:core/setup.py] [bumpversion:file:core/dbt/version.py] - -[bumpversion:file:docker/Dockerfile] From 6e7e55212b74a608c3edeb35419aadcfed82ed15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:05:35 +0100 Subject: [PATCH 03/17] Bump black from 23.3.0 to 24.3.0 (#8074) * Bump black from 23.3.0 to 23.7.0 Bumps [black](https://github.com/psf/black) from 23.3.0 to 23.7.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.3.0...23.7.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Add automated changelog yaml from template for bot PR * Update dev-requirements.txt * Discard changes to .changes/unreleased/Dependencies-20230712-004015.yaml * Add automated changelog yaml from template for bot PR * Update .changes/unreleased/Dependencies-20240410-183321.yaml Co-authored-by: Emily Rockman --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Kshitij Aranke Co-authored-by: Kshitij Aranke Co-authored-by: Emily Rockman --- .changes/unreleased/Dependencies-20240410-183321.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240410-183321.yaml diff --git a/.changes/unreleased/Dependencies-20240410-183321.yaml b/.changes/unreleased/Dependencies-20240410-183321.yaml new file mode 100644 index 00000000000..4cdcd6af835 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240410-183321.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump black from 23.3.0 to >=24.3.0,<25.0" +time: 2024-04-10T18:33:21.00000Z +custom: + Author: dependabot[bot] + PR: 8074 diff --git a/dev-requirements.txt b/dev-requirements.txt index 95925578b2a..da0ee332952 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git@main git+https://github.com/dbt-labs/dbt-postgres.git@main -black==23.3.0 +black>=24.3.0,<25.0 bumpversion ddtrace==2.3.0 docutils From 4c1d0e92cd0a9c09659cc432c016dbc966b89daf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 16 Apr 2024 10:12:21 -0500 Subject: [PATCH 04/17] Fix nightly release & Docker drafts (#9954) * add permission * allow only docker release * move new input to manual runs * log input * check failures/skipped * pr nits --- .github/workflows/nightly-release.yml | 1 + .github/workflows/release.yml | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index c6b5dafaca4..d4f2e5bab15 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -20,6 +20,7 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write # this is the permission that allows Docker release defaults: run: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e1a8a21aad..116dee7cd74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,11 @@ on: type: boolean default: false required: false + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false workflow_call: inputs: target_branch: @@ -81,6 +86,7 @@ jobs: echo The release version number: ${{ inputs.version_number }} echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} + echo Only Docker: ${{ inputs.only_docker }} - name: "Checkout target branch" uses: actions/checkout@v4 @@ -99,6 +105,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog needs: [job-setup] + if: ${{ !inputs.only_docker }} uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main @@ -114,7 +121,7 @@ jobs: log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] @@ -128,7 +135,7 @@ jobs: build-test-package: name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [job-setup, bump-version-generate-changelog] uses: dbt-labs/dbt-release/.github/workflows/build.yml@main @@ -149,7 +156,7 @@ jobs: github-release: name: GitHub Release - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog, build-test-package] @@ -180,6 +187,7 @@ jobs: # dbt-postgres exists within dbt-core for versions 1.7 and earlier but is a separate package for 1.8 and later. # determine if we need to release dbt-core or both dbt-core and dbt-postgres name: Determine Docker Package + if: ${{ !failure() && !cancelled() }} runs-on: ubuntu-latest needs: [pypi-release] outputs: @@ -204,6 +212,10 @@ jobs: docker-release: name: "Docker Release for ${{ matrix.package }}" needs: [determine-docker-package] + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} strategy: matrix: ${{fromJson(needs.determine-docker-package.outputs.matrix)}} From 8b5884b527e539dda4374c9f341fa178db1edda8 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Wed, 17 Apr 2024 01:23:16 +0200 Subject: [PATCH 05/17] Scrub secret vars (#9733) - Scrub secret vars in RequiredVarNotFoundError - Scrub secret vars in StateCheckVarsHash event - Scrub secret vars in run results --- .../unreleased/Features-20240307-153622.yaml | 6 ++ core/dbt/artifacts/schemas/run/v5/run.py | 24 +++++- core/dbt/exceptions.py | 7 +- core/dbt/parser/manifest.py | 7 +- .../context_methods/test_cli_vars.py | 81 ++++++++++++++++++- tests/unit/test_parse_manifest.py | 2 +- 6 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Features-20240307-153622.yaml diff --git a/.changes/unreleased/Features-20240307-153622.yaml b/.changes/unreleased/Features-20240307-153622.yaml new file mode 100644 index 00000000000..80886a82c9b --- /dev/null +++ b/.changes/unreleased/Features-20240307-153622.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support scrubbing secret vars +time: 2024-03-07T15:36:22.754627+01:00 +custom: + Author: nielspardon + Issue: "7247" diff --git a/core/dbt/artifacts/schemas/run/v5/run.py b/core/dbt/artifacts/schemas/run/v5/run.py index 82ef5232c9c..47cc0cb3b87 100644 --- a/core/dbt/artifacts/schemas/run/v5/run.py +++ b/core/dbt/artifacts/schemas/run/v5/run.py @@ -1,9 +1,11 @@ import threading from typing import Any, Optional, Iterable, Tuple, Sequence, Dict, TYPE_CHECKING +import copy from dataclasses import dataclass, field from datetime import datetime +from dbt.constants import SECRET_ENV_PREFIX from dbt.artifacts.resources import CompiledResource from dbt.artifacts.schemas.base import ( BaseArtifactMetadata, @@ -19,6 +21,7 @@ ExecutionResult, ) from dbt_common.clients.system import write_json +from dbt.exceptions import scrub_secrets if TYPE_CHECKING: @@ -123,7 +126,26 @@ def from_execution_results( dbt_schema_version=str(cls.dbt_schema_version), generated_at=generated_at, ) - return cls(metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=args) + + secret_vars = [ + v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() + ] + + scrubbed_args = copy.deepcopy(args) + + # scrub secrets in invocation command + scrubbed_args["invocation_command"] = scrub_secrets( + scrubbed_args["invocation_command"], secret_vars + ) + + # scrub secrets in vars dict + scrubbed_args["vars"] = { + k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items() + } + + return cls( + metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args + ) @classmethod def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 3f3406b43f2..721e65b4c27 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -17,6 +17,8 @@ from dbt_common.dataclass_schema import ValidationError +from dbt.constants import SECRET_ENV_PREFIX + if TYPE_CHECKING: import agate @@ -333,7 +335,10 @@ def get_message(self) -> str: pretty_vars = json.dumps(dct, sort_keys=True, indent=4) msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}" - return msg + return scrub_secrets(msg, self.var_secrets()) + + def var_secrets(self) -> List[str]: + return [v for k, v in self.merged.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()] class PackageNotFoundForMacroError(CompilationError): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 03809635620..a0cc49faa20 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -44,6 +44,7 @@ MANIFEST_FILE_NAME, PARTIAL_PARSE_FILE_NAME, SEMANTIC_MANIFEST_FILE_NAME, + SECRET_ENV_PREFIX, ) from dbt_common.helper_types import PathSet from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error @@ -116,6 +117,7 @@ TargetNotFoundError, AmbiguousAliasError, InvalidAccessTypeError, + scrub_secrets, ) from dbt.parser.base import Parser from dbt.parser.analysis import AnalysisParser @@ -989,6 +991,9 @@ def build_manifest_state_check(self): # of env_vars, that would need to change. # We are using the parsed cli_vars instead of config.args.vars, in order # to sort them and avoid reparsing because of ordering issues. + secret_vars = [ + v for k, v in config.cli_vars.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() + ] stringified_cli_vars = pprint.pformat(config.cli_vars) vars_hash = FileHash.from_contents( "\x00".join( @@ -1003,7 +1008,7 @@ def build_manifest_state_check(self): fire_event( StateCheckVarsHash( checksum=vars_hash.checksum, - vars=stringified_cli_vars, + vars=scrub_secrets(stringified_cli_vars, secret_vars), profile=config.args.profile, target=config.args.target, version=__version__, diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py index d3d5dfc8197..1d72e8c5021 100644 --- a/tests/functional/context_methods/test_cli_vars.py +++ b/tests/functional/context_methods/test_cli_vars.py @@ -3,7 +3,13 @@ from tests.fixtures.dbt_integration_project import dbt_integration_project # noqa: F401 -from dbt.tests.util import run_dbt, get_artifact, write_config_file +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, + get_logging_events, + get_artifact, + write_config_file, +) from dbt.tests.fixtures.project import write_project_files from dbt.exceptions import DbtRuntimeError, CompilationError @@ -206,3 +212,76 @@ def test_vars_in_selectors(self, project): # Var in cli_vars works results = run_dbt(["run", "--vars", "snapshot_target: dev"]) assert len(results) == 1 + + +models_scrubbing__schema_yml = """ +version: 2 +models: +- name: simple_model + columns: + - name: simple + data_tests: + - accepted_values: + values: + - abc +""" + +models_scrubbing__simple_model_sql = """ +select + '{{ var("DBT_ENV_SECRET_simple") }}'::varchar as simple +""" + + +class TestCLIVarsScrubbing: + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models_scrubbing__schema_yml, + "simple_model.sql": models_scrubbing__simple_model_sql, + } + + def test__run_results_scrubbing(self, project): + results, output = run_dbt_and_capture( + [ + "--debug", + "--log-format", + "json", + "run", + "--vars", + "{DBT_ENV_SECRET_simple: abc, unused: def}", + ] + ) + assert len(results) == 1 + + run_results = get_artifact(project.project_root, "target", "run_results.json") + assert run_results["args"]["vars"] == { + "DBT_ENV_SECRET_simple": "*****", + "unused": "def", + } + + log_events = get_logging_events(log_output=output, event_name="StateCheckVarsHash") + assert len(log_events) == 1 + assert ( + log_events[0]["data"]["vars"] == "{'DBT_ENV_SECRET_simple': '*****', 'unused': 'def'}" + ) + + def test__exception_scrubbing(self, project): + results, output = run_dbt_and_capture( + [ + "--debug", + "--log-format", + "json", + "run", + "--vars", + "{DBT_ENV_SECRET_unused: abc, unused: def}", + ], + False, + ) + assert len(results) == 1 + + log_events = get_logging_events(log_output=output, event_name="CatchableExceptionOnRun") + assert len(log_events) == 1 + assert ( + '{\n "DBT_ENV_SECRET_unused": "*****",\n "unused": "def"\n }' + in log_events[0]["info"]["msg"] + ) diff --git a/tests/unit/test_parse_manifest.py b/tests/unit/test_parse_manifest.py index 23d6d51446d..7471b399acd 100644 --- a/tests/unit/test_parse_manifest.py +++ b/tests/unit/test_parse_manifest.py @@ -108,7 +108,7 @@ def _new_file(self, searched, name, match): class TestPartialParse(unittest.TestCase): def setUp(self) -> None: mock_project = MagicMock(RuntimeConfig) - mock_project.cli_vars = "" + mock_project.cli_vars = {} mock_project.args = MagicMock() mock_project.args.profile = "test" mock_project.args.target = "test" From a70024f74539448172029b74a8321ef9d1f44514 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Tue, 16 Apr 2024 16:36:05 -0700 Subject: [PATCH 06/17] Reorganize fixtures and implement a happy path test for semantic_manifest (#9930) --- .../Under the Hood-20240416-150030.yaml | 6 + .../artifacts/resources/v1/semantic_model.py | 2 +- tests/unit/conftest.py | 3 + tests/unit/contracts/__init__.py | 0 tests/unit/contracts/graph/__init__.py | 0 .../contracts/graph/test_semantic_manifest.py | 28 + tests/unit/test_graph_selector_methods.py | 915 +-------------- tests/unit/{utils.py => utils/__init__.py} | 0 tests/unit/utils/manifest.py | 1006 +++++++++++++++++ 9 files changed, 1061 insertions(+), 899 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240416-150030.yaml create mode 100644 tests/unit/contracts/__init__.py create mode 100644 tests/unit/contracts/graph/__init__.py create mode 100644 tests/unit/contracts/graph/test_semantic_manifest.py rename tests/unit/{utils.py => utils/__init__.py} (100%) create mode 100644 tests/unit/utils/manifest.py diff --git a/.changes/unreleased/Under the Hood-20240416-150030.yaml b/.changes/unreleased/Under the Hood-20240416-150030.yaml new file mode 100644 index 00000000000..b57a01a6cc6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240416-150030.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add a test for semantic manifest and move test fixtures needed for it +time: 2024-04-16T15:00:30.614286-07:00 +custom: + Author: ChenyuLInx + Issue: "9665" diff --git a/core/dbt/artifacts/resources/v1/semantic_model.py b/core/dbt/artifacts/resources/v1/semantic_model.py index b219b2bdcc8..8a02aa5fa61 100644 --- a/core/dbt/artifacts/resources/v1/semantic_model.py +++ b/core/dbt/artifacts/resources/v1/semantic_model.py @@ -42,7 +42,7 @@ class NodeRelation(dbtClassMixin): alias: str schema_name: str # TODO: Could this be called simply "schema" so we could reuse StateRelation? database: Optional[str] = None - relation_name: Optional[str] = None + relation_name: Optional[str] = "" # ==================================== diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 6f45963cb78..5e9acb84907 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -4,6 +4,9 @@ from dbt.artifacts.resources.types import NodeType from dbt.contracts.graph.nodes import SourceDefinition +# All manifest related fixtures. +from tests.unit.utils.manifest import * # noqa + @pytest.fixture def basic_parsed_source_definition_object(): diff --git a/tests/unit/contracts/__init__.py b/tests/unit/contracts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/contracts/graph/__init__.py b/tests/unit/contracts/graph/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/contracts/graph/test_semantic_manifest.py b/tests/unit/contracts/graph/test_semantic_manifest.py new file mode 100644 index 00000000000..4eb389ea0f5 --- /dev/null +++ b/tests/unit/contracts/graph/test_semantic_manifest.py @@ -0,0 +1,28 @@ +import pytest +from dbt.contracts.graph.semantic_manifest import SemanticManifest + + +# Overwrite the default nods to construct the manifest +@pytest.fixture +def nodes(metricflow_time_spine_model): + return [metricflow_time_spine_model] + + +@pytest.fixture +def semantic_models( + semantic_model, +) -> list: + return [semantic_model] + + +@pytest.fixture +def metrics( + metric, +) -> list: + return [metric] + + +class TestSemanticManifest: + def test_validate(self, manifest): + sm_manifest = SemanticManifest(manifest) + assert sm_manifest.validate() diff --git a/tests/unit/test_graph_selector_methods.py b/tests/unit/test_graph_selector_methods.py index 52013fd409d..1a3a16fdafc 100644 --- a/tests/unit/test_graph_selector_methods.py +++ b/tests/unit/test_graph_selector_methods.py @@ -1,49 +1,13 @@ -from argparse import Namespace import copy -from dataclasses import replace import pytest +from dataclasses import replace from unittest import mock from pathlib import Path +from dbt.artifacts.resources import ColumnInfo, FileHash +from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.files import FileHash -from dbt.contracts.graph.nodes import ( - DependsOn, - NodeConfig, - Macro, - ModelNode, - Exposure, - Metric, - Group, - SavedQuery, - SeedNode, - SemanticModel, - SingularTestNode, - GenericTestNode, - SourceDefinition, - AccessType, - UnitTestDefinition, -) -from dbt.contracts.graph.manifest import Manifest, ManifestMetadata -from dbt.artifacts.resources import ( - ColumnInfo, - ExposureType, - MetricInputMeasure, - MetricTypeParams, - NodeRelation, - Owner, - QueryParams, - MacroDependsOn, - TestConfig, - TestMetadata, - RefArgs, -) -from dbt.contracts.graph.unparsed import ( - UnitTestInputFixture, - UnitTestOutputFixture, -) from dbt.contracts.state import PreviousState -from dbt.node_types import NodeType from dbt.graph.selector_methods import ( MethodManager, QualifiedNameSelectorMethod, @@ -65,862 +29,17 @@ SemanticModelSelectorMethod, ) import dbt_common.exceptions -from dbt_semantic_interfaces.type_enums import MetricType from .utils import replace_config -from dbt.flags import set_from_args - -set_from_args(Namespace(WARN_ERROR=False), None) - - -def make_model( - pkg, - name, - sql, - refs=None, - sources=None, - tags=None, - path=None, - alias=None, - config_kwargs=None, - fqn_extras=None, - depends_on_macros=None, - version=None, - latest_version=None, - access=None, -): - if refs is None: - refs = [] - if sources is None: - sources = [] - if tags is None: - tags = [] - if path is None: - path = f"{name}.sql" - if alias is None: - alias = name - if config_kwargs is None: - config_kwargs = {} - if depends_on_macros is None: - depends_on_macros = [] - - if fqn_extras is None: - fqn_extras = [] - - fqn = [pkg] + fqn_extras + [name] - if version: - fqn.append(f"v{version}") - - depends_on_nodes = [] - source_values = [] - ref_values = [] - for ref in refs: - ref_version = ref.version if hasattr(ref, "version") else None - ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) - depends_on_nodes.append(ref.unique_id) - for src in sources: - source_values.append([src.source_name, src.name]) - depends_on_nodes.append(src.unique_id) - - return ModelNode( - language="sql", - raw_code=sql, - database="dbt", - schema="dbt_schema", - alias=alias, - name=name, - fqn=fqn, - unique_id=f"model.{pkg}.{name}" if not version else f"model.{pkg}.{name}.v{version}", - package_name=pkg, - path=path, - original_file_path=f"models/{path}", - config=NodeConfig(**config_kwargs), - tags=tags, - refs=ref_values, - sources=source_values, - depends_on=DependsOn( - nodes=depends_on_nodes, - macros=depends_on_macros, - ), - resource_type=NodeType.Model, - checksum=FileHash.from_contents(""), - version=version, - latest_version=latest_version, - access=access or AccessType.Protected, - ) - - -def make_seed( - pkg, name, path=None, loader=None, alias=None, tags=None, fqn_extras=None, checksum=None -): - if alias is None: - alias = name - if tags is None: - tags = [] - if path is None: - path = f"{name}.csv" - - if fqn_extras is None: - fqn_extras = [] - - if checksum is None: - checksum = FileHash.from_contents("") - - fqn = [pkg] + fqn_extras + [name] - return SeedNode( - database="dbt", - schema="dbt_schema", - alias=alias, - name=name, - fqn=fqn, - unique_id=f"seed.{pkg}.{name}", - package_name=pkg, - path=path, - original_file_path=f"data/{path}", - tags=tags, - resource_type=NodeType.Seed, - checksum=FileHash.from_contents(""), - ) - - -def make_source( - pkg, source_name, table_name, path=None, loader=None, identifier=None, fqn_extras=None -): - if path is None: - path = "models/schema.yml" - if loader is None: - loader = "my_loader" - if identifier is None: - identifier = table_name - - if fqn_extras is None: - fqn_extras = [] - - fqn = [pkg] + fqn_extras + [source_name, table_name] - - return SourceDefinition( - fqn=fqn, - database="dbt", - schema="dbt_schema", - unique_id=f"source.{pkg}.{source_name}.{table_name}", - package_name=pkg, - path=path, - original_file_path=path, - name=table_name, - source_name=source_name, - loader="my_loader", - identifier=identifier, - resource_type=NodeType.Source, - loaded_at_field="loaded_at", - tags=[], - source_description="", - ) - - -def make_macro(pkg, name, macro_sql, path=None, depends_on_macros=None): - if path is None: - path = "macros/macros.sql" - - if depends_on_macros is None: - depends_on_macros = [] - - return Macro( - name=name, - macro_sql=macro_sql, - unique_id=f"macro.{pkg}.{name}", - package_name=pkg, - path=path, - original_file_path=path, - resource_type=NodeType.Macro, - depends_on=MacroDependsOn(macros=depends_on_macros), - ) - - -def make_unique_test(pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None): - return make_generic_test(pkg, "unique", test_model, {}, column_name=column_name) - - -def make_not_null_test( - pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None -): - return make_generic_test(pkg, "not_null", test_model, {}, column_name=column_name) - - -def make_generic_test( - pkg, - test_name, - test_model, - test_kwargs, - path=None, - refs=None, - sources=None, - tags=None, - column_name=None, -): - kwargs = test_kwargs.copy() - ref_values = [] - source_values = [] - # this doesn't really have to be correct - if isinstance(test_model, SourceDefinition): - kwargs["model"] = ( - "{{ source('" + test_model.source_name + "', '" + test_model.name + "') }}" - ) - source_values.append([test_model.source_name, test_model.name]) - else: - kwargs["model"] = "{{ ref('" + test_model.name + "')}}" - ref_values.append( - RefArgs( - name=test_model.name, package=test_model.package_name, version=test_model.version - ) - ) - if column_name is not None: - kwargs["column_name"] = column_name - - # whatever - args_name = test_model.search_name.replace(".", "_") - if column_name is not None: - args_name += "_" + column_name - node_name = f"{test_name}_{args_name}" - raw_code = ( - '{{ config(severity="ERROR") }}{{ test_' + test_name + "(**dbt_schema_test_kwargs) }}" - ) - name_parts = test_name.split(".") - - if len(name_parts) == 2: - namespace, test_name = name_parts - macro_depends = f"macro.{namespace}.test_{test_name}" - elif len(name_parts) == 1: - namespace = None - macro_depends = f"macro.dbt.test_{test_name}" - else: - assert False, f"invalid test name: {test_name}" - - if path is None: - path = "schema.yml" - if tags is None: - tags = ["schema"] - - if refs is None: - refs = [] - if sources is None: - sources = [] - - depends_on_nodes = [] - for ref in refs: - ref_version = ref.version if hasattr(ref, "version") else None - ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) - depends_on_nodes.append(ref.unique_id) - - for source in sources: - source_values.append([source.source_name, source.name]) - depends_on_nodes.append(source.unique_id) - - return GenericTestNode( - language="sql", - raw_code=raw_code, - test_metadata=TestMetadata( - namespace=namespace, - name=test_name, - kwargs=kwargs, - ), - database="dbt", - schema="dbt_postgres", - name=node_name, - alias=node_name, - fqn=["minimal", "schema_test", node_name], - unique_id=f"test.{pkg}.{node_name}", - package_name=pkg, - path=f"schema_test/{node_name}.sql", - original_file_path=f"models/{path}", - resource_type=NodeType.Test, - tags=tags, - refs=ref_values, - sources=[], - depends_on=DependsOn(macros=[macro_depends], nodes=depends_on_nodes), - column_name=column_name, - checksum=FileHash.from_contents(""), - ) - - -def make_unit_test( - pkg, - test_name, - test_model, -): - input_fixture = UnitTestInputFixture( - input="ref('table_model')", - rows=[{"id": 1, "string_a": "a"}], - ) - output_fixture = UnitTestOutputFixture( - rows=[{"id": 1, "string_a": "a"}], - ) - return UnitTestDefinition( - name=test_name, - model=test_model, - package_name=pkg, - resource_type=NodeType.Unit, - path="unit_tests.yml", - original_file_path="models/unit_tests.yml", - unique_id=f"unit.{pkg}.{test_model.name}__{test_name}", - given=[input_fixture], - expect=output_fixture, - fqn=[pkg, test_model.name, test_name], - ) - - -def make_singular_test( - pkg, name, sql, refs=None, sources=None, tags=None, path=None, config_kwargs=None -): - - if refs is None: - refs = [] - if sources is None: - sources = [] - if tags is None: - tags = ["data"] - if path is None: - path = f"{name}.sql" - - if config_kwargs is None: - config_kwargs = {} - - fqn = ["minimal", "data_test", name] - - depends_on_nodes = [] - source_values = [] - ref_values = [] - for ref in refs: - ref_version = ref.version if hasattr(ref, "version") else None - ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) - depends_on_nodes.append(ref.unique_id) - for src in sources: - source_values.append([src.source_name, src.name]) - depends_on_nodes.append(src.unique_id) - - return SingularTestNode( - language="sql", - raw_code=sql, - database="dbt", - schema="dbt_schema", - name=name, - alias=name, - fqn=fqn, - unique_id=f"test.{pkg}.{name}", - package_name=pkg, - path=path, - original_file_path=f"tests/{path}", - config=TestConfig(**config_kwargs), - tags=tags, - refs=ref_values, - sources=source_values, - depends_on=DependsOn(nodes=depends_on_nodes, macros=[]), - resource_type=NodeType.Test, - checksum=FileHash.from_contents(""), - ) - - -def make_exposure(pkg, name, path=None, fqn_extras=None, owner=None): - if path is None: - path = "schema.yml" - - if fqn_extras is None: - fqn_extras = [] - - if owner is None: - owner = Owner(email="test@example.com") - - fqn = [pkg, "exposures"] + fqn_extras + [name] - return Exposure( - name=name, - resource_type=NodeType.Exposure, - type=ExposureType.Notebook, - fqn=fqn, - unique_id=f"exposure.{pkg}.{name}", - package_name=pkg, - path=path, - original_file_path=path, - owner=owner, - ) - - -def make_metric(pkg, name, path=None): - if path is None: - path = "schema.yml" - - return Metric( - name=name, - resource_type=NodeType.Metric, - path=path, - package_name=pkg, - original_file_path=path, - unique_id=f"metric.{pkg}.{name}", - fqn=[pkg, "metrics", name], - label="New Customers", - description="New customers", - type=MetricType.SIMPLE, - type_params=MetricTypeParams(measure=MetricInputMeasure(name="count_cats")), - meta={"is_okr": True}, - tags=["okrs"], - ) - - -def make_group(pkg, name, path=None): - if path is None: - path = "schema.yml" - - return Group( - name=name, - resource_type=NodeType.Group, - path=path, - package_name=pkg, - original_file_path=path, - unique_id=f"group.{pkg}.{name}", - owner="email@gmail.com", - ) - - -def make_semantic_model(pkg: str, name: str, path=None, model=None): - if path is None: - path = "schema.yml" - - if model is None: - model = name - - node_relation = NodeRelation( - alias=model, - schema_name="dbt", - ) - - return SemanticModel( - name=name, - resource_type=NodeType.SemanticModel, - model=model, - node_relation=node_relation, - package_name=pkg, - path=path, - description="Customer entity", - primary_entity="customer", - unique_id=f"semantic_model.{pkg}.{name}", - original_file_path=path, - fqn=[pkg, "semantic_models", name], - ) - - -def make_saved_query(pkg: str, name: str, metric: str, path=None): - if path is None: - path = "schema.yml" - - return SavedQuery( - name=name, - resource_type=NodeType.SavedQuery, - package_name=pkg, - path=path, - description="Test Saved Query", - query_params=QueryParams( - metrics=[metric], - group_by=[], - where=None, - ), - exports=[], - unique_id=f"saved_query.{pkg}.{name}", - original_file_path=path, - fqn=[pkg, "saved_queries", name], - ) - - -@pytest.fixture -def macro_test_unique(): - return make_macro( - "dbt", "test_unique", "blablabla", depends_on_macros=["macro.dbt.default__test_unique"] - ) - - -@pytest.fixture -def macro_default_test_unique(): - return make_macro("dbt", "default__test_unique", "blablabla") - - -@pytest.fixture -def macro_test_not_null(): - return make_macro( - "dbt", "test_not_null", "blablabla", depends_on_macros=["macro.dbt.default__test_not_null"] - ) - - -@pytest.fixture -def macro_default_test_not_null(): - return make_macro("dbt", "default__test_not_null", "blabla") - - -@pytest.fixture -def seed(): - return make_seed("pkg", "seed") - - -@pytest.fixture -def source(): - return make_source("pkg", "raw", "seed", identifier="seed") - - -@pytest.fixture -def ephemeral_model(source): - return make_model( - "pkg", - "ephemeral_model", - 'select * from {{ source("raw", "seed") }}', - config_kwargs={"materialized": "ephemeral"}, - sources=[source], - ) - - -@pytest.fixture -def view_model(ephemeral_model): - return make_model( - "pkg", - "view_model", - 'select * from {{ ref("ephemeral_model") }}', - config_kwargs={"materialized": "view"}, - refs=[ephemeral_model], - tags=["uses_ephemeral"], - ) - - -@pytest.fixture -def table_model(ephemeral_model): - return make_model( - "pkg", - "table_model", - 'select * from {{ ref("ephemeral_model") }}', - config_kwargs={ - "materialized": "table", - "meta": { - # Other properties to test in test_select_config_meta - "string_property": "some_string", - "truthy_bool_property": True, - "falsy_bool_property": False, - "list_property": ["some_value", True, False], - }, - }, - refs=[ephemeral_model], - tags=["uses_ephemeral"], - path="subdirectory/table_model.sql", - ) - - -@pytest.fixture -def table_model_py(seed): - return make_model( - "pkg", - "table_model_py", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - tags=[], - path="subdirectory/table_model.py", - ) - - -@pytest.fixture -def table_model_csv(seed): - return make_model( - "pkg", - "table_model_csv", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - tags=[], - path="subdirectory/table_model.csv", - ) - - -@pytest.fixture -def ext_source(): - return make_source( - "ext", - "ext_raw", - "ext_source", - ) - - -@pytest.fixture -def ext_source_2(): - return make_source( - "ext", - "ext_raw", - "ext_source_2", - ) - - -@pytest.fixture -def ext_source_other(): - return make_source( - "ext", - "raw", - "ext_source", - ) - - -@pytest.fixture -def ext_source_other_2(): - return make_source( - "ext", - "raw", - "ext_source_2", - ) - - -@pytest.fixture -def ext_model(ext_source): - return make_model( - "ext", - "ext_model", - 'select * from {{ source("ext_raw", "ext_source") }}', - sources=[ext_source], - ) - - -@pytest.fixture -def union_model(seed, ext_source): - return make_model( - "pkg", - "union_model", - 'select * from {{ ref("seed") }} union all select * from {{ source("ext_raw", "ext_source") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[ext_source], - fqn_extras=["unions"], - path="subdirectory/union_model.sql", - tags=["unions"], - ) - - -@pytest.fixture -def versioned_model_v1(seed): - return make_model( - "pkg", - "versioned_model", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[], - path="subdirectory/versioned_model_v1.sql", - version=1, - latest_version=2, - ) - - -@pytest.fixture -def versioned_model_v2(seed): - return make_model( - "pkg", - "versioned_model", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[], - path="subdirectory/versioned_model_v2.sql", - version=2, - latest_version=2, - ) - - -@pytest.fixture -def versioned_model_v3(seed): - return make_model( - "pkg", - "versioned_model", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[], - path="subdirectory/versioned_model_v3.sql", - version="3", - latest_version=2, - ) - - -@pytest.fixture -def versioned_model_v12_string(seed): - return make_model( - "pkg", - "versioned_model", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[], - path="subdirectory/versioned_model_v12.sql", - version="12", - latest_version=2, - ) - - -@pytest.fixture -def versioned_model_v4_nested_dir(seed): - return make_model( - "pkg", - "versioned_model", - 'select * from {{ ref("seed") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[], - path="subdirectory/nested_dir/versioned_model_v3.sql", - version="4", - latest_version=2, - fqn_extras=["nested_dir"], - ) - - -@pytest.fixture -def table_id_unique(table_model): - return make_unique_test("pkg", table_model, "id") - - -@pytest.fixture -def table_id_not_null(table_model): - return make_not_null_test("pkg", table_model, "id") - - -@pytest.fixture -def view_id_unique(view_model): - return make_unique_test("pkg", view_model, "id") - - -@pytest.fixture -def ext_source_id_unique(ext_source): - return make_unique_test("ext", ext_source, "id") - - -@pytest.fixture -def view_test_nothing(view_model): - return make_singular_test( - "pkg", - "view_test_nothing", - 'select * from {{ ref("view_model") }} limit 0', - refs=[view_model], - ) - - -@pytest.fixture -def unit_test_table_model(table_model): - return make_unit_test( - "pkg", - "unit_test_table_model", - table_model, - ) - - -# Support dots as namespace separators -@pytest.fixture -def namespaced_seed(): - return make_seed("pkg", "mynamespace.seed") - - -@pytest.fixture -def namespace_model(source): - return make_model( - "pkg", - "mynamespace.ephemeral_model", - 'select * from {{ source("raw", "seed") }}', - config_kwargs={"materialized": "ephemeral"}, - sources=[source], - ) - - -@pytest.fixture -def namespaced_union_model(seed, ext_source): - return make_model( - "pkg", - "mynamespace.union_model", - 'select * from {{ ref("mynamespace.seed") }} union all select * from {{ ref("mynamespace.ephemeral_model") }}', - config_kwargs={"materialized": "table"}, - refs=[seed], - sources=[ext_source], - fqn_extras=["unions"], - path="subdirectory/union_model.sql", - tags=["unions"], - ) - - -@pytest.fixture -def manifest( - seed, - source, - ephemeral_model, - view_model, - table_model, - table_model_py, - table_model_csv, - ext_source, - ext_model, - union_model, - versioned_model_v1, - versioned_model_v2, - versioned_model_v3, - versioned_model_v4_nested_dir, - versioned_model_v12_string, - ext_source_2, - ext_source_other, - ext_source_other_2, - table_id_unique, - table_id_not_null, - view_id_unique, - ext_source_id_unique, - view_test_nothing, - namespaced_seed, - namespace_model, - namespaced_union_model, - macro_test_unique, - macro_default_test_unique, - macro_test_not_null, - macro_default_test_not_null, - unit_test_table_model, -): - nodes = [ - seed, - ephemeral_model, - view_model, - table_model, - table_model_py, - table_model_csv, - union_model, - versioned_model_v1, - versioned_model_v2, - versioned_model_v3, - versioned_model_v4_nested_dir, - versioned_model_v12_string, - ext_model, - table_id_unique, - table_id_not_null, - view_id_unique, - ext_source_id_unique, - view_test_nothing, - namespaced_seed, - namespace_model, - namespaced_union_model, - ] - sources = [source, ext_source, ext_source_2, ext_source_other, ext_source_other_2] - macros = [ - macro_test_unique, - macro_default_test_unique, - macro_test_not_null, - macro_default_test_not_null, - ] - unit_tests = [unit_test_table_model] - manifest = Manifest( - nodes={n.unique_id: n for n in nodes}, - sources={s.unique_id: s for s in sources}, - macros={m.unique_id: m for m in macros}, - unit_tests={t.unique_id: t for t in unit_tests}, - semantic_models={}, - docs={}, - files={}, - exposures={}, - metrics={}, - disabled={}, - selectors={}, - groups={}, - metadata=ManifestMetadata(adapter_type="postgres"), - ) - return manifest +from tests.unit.utils.manifest import ( + make_model, + make_seed, + make_exposure, + make_metric, + make_saved_query, + make_semantic_model, + make_group, + make_macro, +) def search_manifest_using_method(manifest, method, selection): @@ -1373,11 +492,11 @@ def test_select_metric(manifest): assert search_manifest_using_method(manifest, method, "*_metric") == {"my_metric"} -def test_select_semantic_model(manifest): +def test_select_semantic_model(manifest, table_model): semantic_model = make_semantic_model( "pkg", "customer", - model="customers", + model=table_model, path="_semantic_models.yml", ) manifest.semantic_models[semantic_model.unique_id] = semantic_model @@ -1389,11 +508,11 @@ def test_select_semantic_model(manifest): assert search_manifest_using_method(manifest, method, "*omer") == {"customer"} -def test_select_semantic_model_by_tag(manifest): +def test_select_semantic_model_by_tag(manifest, table_model): semantic_model = make_semantic_model( "pkg", "customer", - model="customers", + model=table_model, path="_semantic_models.yml", ) manifest.semantic_models[semantic_model.unique_id] = semantic_model diff --git a/tests/unit/utils.py b/tests/unit/utils/__init__.py similarity index 100% rename from tests/unit/utils.py rename to tests/unit/utils/__init__.py diff --git a/tests/unit/utils/manifest.py b/tests/unit/utils/manifest.py new file mode 100644 index 00000000000..2f56570df41 --- /dev/null +++ b/tests/unit/utils/manifest.py @@ -0,0 +1,1006 @@ +from argparse import Namespace +import pytest + +from dbt.artifacts.resources.v1.model import ModelConfig +from dbt.contracts.files import FileHash +from dbt.contracts.graph.nodes import ( + DependsOn, + NodeConfig, + Macro, + ModelNode, + Exposure, + Metric, + Group, + SavedQuery, + SeedNode, + SemanticModel, + SingularTestNode, + GenericTestNode, + SourceDefinition, + AccessType, + UnitTestDefinition, +) +from dbt.contracts.graph.manifest import Manifest, ManifestMetadata +from dbt.artifacts.resources import ( + ExposureType, + MetricInputMeasure, + MetricTypeParams, + NodeRelation, + Owner, + QueryParams, + MacroDependsOn, + TestConfig, + TestMetadata, + RefArgs, +) +from dbt.contracts.graph.unparsed import ( + UnitTestInputFixture, + UnitTestOutputFixture, +) +from dbt.node_types import NodeType + +from dbt_semantic_interfaces.type_enums import MetricType +from dbt.flags import set_from_args + +set_from_args(Namespace(WARN_ERROR=False), None) + + +def make_model( + pkg, + name, + sql, + refs=None, + sources=None, + tags=None, + path=None, + alias=None, + config_kwargs=None, + fqn_extras=None, + depends_on_macros=None, + version=None, + latest_version=None, + access=None, +): + if refs is None: + refs = [] + if sources is None: + sources = [] + if tags is None: + tags = [] + if path is None: + path = f"{name}.sql" + if alias is None: + alias = name + if config_kwargs is None: + config_kwargs = {} + if depends_on_macros is None: + depends_on_macros = [] + + if fqn_extras is None: + fqn_extras = [] + + fqn = [pkg] + fqn_extras + [name] + if version: + fqn.append(f"v{version}") + + depends_on_nodes = [] + source_values = [] + ref_values = [] + for ref in refs: + ref_version = ref.version if hasattr(ref, "version") else None + ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) + depends_on_nodes.append(ref.unique_id) + for src in sources: + source_values.append([src.source_name, src.name]) + depends_on_nodes.append(src.unique_id) + + return ModelNode( + language="sql", + raw_code=sql, + database="dbt", + schema="dbt_schema", + alias=alias, + name=name, + fqn=fqn, + unique_id=f"model.{pkg}.{name}" if not version else f"model.{pkg}.{name}.v{version}", + package_name=pkg, + path=path, + original_file_path=f"models/{path}", + config=NodeConfig(**config_kwargs), + tags=tags, + refs=ref_values, + sources=source_values, + depends_on=DependsOn( + nodes=depends_on_nodes, + macros=depends_on_macros, + ), + resource_type=NodeType.Model, + checksum=FileHash.from_contents(""), + version=version, + latest_version=latest_version, + access=access or AccessType.Protected, + ) + + +def make_seed( + pkg, name, path=None, loader=None, alias=None, tags=None, fqn_extras=None, checksum=None +): + if alias is None: + alias = name + if tags is None: + tags = [] + if path is None: + path = f"{name}.csv" + + if fqn_extras is None: + fqn_extras = [] + + if checksum is None: + checksum = FileHash.from_contents("") + + fqn = [pkg] + fqn_extras + [name] + return SeedNode( + database="dbt", + schema="dbt_schema", + alias=alias, + name=name, + fqn=fqn, + unique_id=f"seed.{pkg}.{name}", + package_name=pkg, + path=path, + original_file_path=f"data/{path}", + tags=tags, + resource_type=NodeType.Seed, + checksum=FileHash.from_contents(""), + ) + + +def make_source( + pkg, source_name, table_name, path=None, loader=None, identifier=None, fqn_extras=None +): + if path is None: + path = "models/schema.yml" + if loader is None: + loader = "my_loader" + if identifier is None: + identifier = table_name + + if fqn_extras is None: + fqn_extras = [] + + fqn = [pkg] + fqn_extras + [source_name, table_name] + + return SourceDefinition( + fqn=fqn, + database="dbt", + schema="dbt_schema", + unique_id=f"source.{pkg}.{source_name}.{table_name}", + package_name=pkg, + path=path, + original_file_path=path, + name=table_name, + source_name=source_name, + loader="my_loader", + identifier=identifier, + resource_type=NodeType.Source, + loaded_at_field="loaded_at", + tags=[], + source_description="", + ) + + +def make_macro(pkg, name, macro_sql, path=None, depends_on_macros=None): + if path is None: + path = "macros/macros.sql" + + if depends_on_macros is None: + depends_on_macros = [] + + return Macro( + name=name, + macro_sql=macro_sql, + unique_id=f"macro.{pkg}.{name}", + package_name=pkg, + path=path, + original_file_path=path, + resource_type=NodeType.Macro, + depends_on=MacroDependsOn(macros=depends_on_macros), + ) + + +def make_unique_test(pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None): + return make_generic_test(pkg, "unique", test_model, {}, column_name=column_name) + + +def make_not_null_test( + pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None +): + return make_generic_test(pkg, "not_null", test_model, {}, column_name=column_name) + + +def make_generic_test( + pkg, + test_name, + test_model, + test_kwargs, + path=None, + refs=None, + sources=None, + tags=None, + column_name=None, +): + kwargs = test_kwargs.copy() + ref_values = [] + source_values = [] + # this doesn't really have to be correct + if isinstance(test_model, SourceDefinition): + kwargs["model"] = ( + "{{ source('" + test_model.source_name + "', '" + test_model.name + "') }}" + ) + source_values.append([test_model.source_name, test_model.name]) + else: + kwargs["model"] = "{{ ref('" + test_model.name + "')}}" + ref_values.append( + RefArgs( + name=test_model.name, package=test_model.package_name, version=test_model.version + ) + ) + if column_name is not None: + kwargs["column_name"] = column_name + + # whatever + args_name = test_model.search_name.replace(".", "_") + if column_name is not None: + args_name += "_" + column_name + node_name = f"{test_name}_{args_name}" + raw_code = ( + '{{ config(severity="ERROR") }}{{ test_' + test_name + "(**dbt_schema_test_kwargs) }}" + ) + name_parts = test_name.split(".") + + if len(name_parts) == 2: + namespace, test_name = name_parts + macro_depends = f"macro.{namespace}.test_{test_name}" + elif len(name_parts) == 1: + namespace = None + macro_depends = f"macro.dbt.test_{test_name}" + else: + assert False, f"invalid test name: {test_name}" + + if path is None: + path = "schema.yml" + if tags is None: + tags = ["schema"] + + if refs is None: + refs = [] + if sources is None: + sources = [] + + depends_on_nodes = [] + for ref in refs: + ref_version = ref.version if hasattr(ref, "version") else None + ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) + depends_on_nodes.append(ref.unique_id) + + for source in sources: + source_values.append([source.source_name, source.name]) + depends_on_nodes.append(source.unique_id) + + return GenericTestNode( + language="sql", + raw_code=raw_code, + test_metadata=TestMetadata( + namespace=namespace, + name=test_name, + kwargs=kwargs, + ), + database="dbt", + schema="dbt_postgres", + name=node_name, + alias=node_name, + fqn=["minimal", "schema_test", node_name], + unique_id=f"test.{pkg}.{node_name}", + package_name=pkg, + path=f"schema_test/{node_name}.sql", + original_file_path=f"models/{path}", + resource_type=NodeType.Test, + tags=tags, + refs=ref_values, + sources=[], + depends_on=DependsOn(macros=[macro_depends], nodes=depends_on_nodes), + column_name=column_name, + checksum=FileHash.from_contents(""), + ) + + +def make_unit_test( + pkg, + test_name, + test_model, +): + input_fixture = UnitTestInputFixture( + input="ref('table_model')", + rows=[{"id": 1, "string_a": "a"}], + ) + output_fixture = UnitTestOutputFixture( + rows=[{"id": 1, "string_a": "a"}], + ) + return UnitTestDefinition( + name=test_name, + model=test_model, + package_name=pkg, + resource_type=NodeType.Unit, + path="unit_tests.yml", + original_file_path="models/unit_tests.yml", + unique_id=f"unit.{pkg}.{test_model.name}__{test_name}", + given=[input_fixture], + expect=output_fixture, + fqn=[pkg, test_model.name, test_name], + ) + + +def make_singular_test( + pkg, name, sql, refs=None, sources=None, tags=None, path=None, config_kwargs=None +): + if refs is None: + refs = [] + if sources is None: + sources = [] + if tags is None: + tags = ["data"] + if path is None: + path = f"{name}.sql" + + if config_kwargs is None: + config_kwargs = {} + + fqn = ["minimal", "data_test", name] + + depends_on_nodes = [] + source_values = [] + ref_values = [] + for ref in refs: + ref_version = ref.version if hasattr(ref, "version") else None + ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) + depends_on_nodes.append(ref.unique_id) + for src in sources: + source_values.append([src.source_name, src.name]) + depends_on_nodes.append(src.unique_id) + + return SingularTestNode( + language="sql", + raw_code=sql, + database="dbt", + schema="dbt_schema", + name=name, + alias=name, + fqn=fqn, + unique_id=f"test.{pkg}.{name}", + package_name=pkg, + path=path, + original_file_path=f"tests/{path}", + config=TestConfig(**config_kwargs), + tags=tags, + refs=ref_values, + sources=source_values, + depends_on=DependsOn(nodes=depends_on_nodes, macros=[]), + resource_type=NodeType.Test, + checksum=FileHash.from_contents(""), + ) + + +def make_exposure(pkg, name, path=None, fqn_extras=None, owner=None): + if path is None: + path = "schema.yml" + + if fqn_extras is None: + fqn_extras = [] + + if owner is None: + owner = Owner(email="test@example.com") + + fqn = [pkg, "exposures"] + fqn_extras + [name] + return Exposure( + name=name, + resource_type=NodeType.Exposure, + type=ExposureType.Notebook, + fqn=fqn, + unique_id=f"exposure.{pkg}.{name}", + package_name=pkg, + path=path, + original_file_path=path, + owner=owner, + ) + + +def make_metric(pkg, name, path=None): + if path is None: + path = "schema.yml" + + return Metric( + name=name, + resource_type=NodeType.Metric, + path=path, + package_name=pkg, + original_file_path=path, + unique_id=f"metric.{pkg}.{name}", + fqn=[pkg, "metrics", name], + label="New Customers", + description="New customers", + type=MetricType.SIMPLE, + type_params=MetricTypeParams(measure=MetricInputMeasure(name="count_cats")), + meta={"is_okr": True}, + tags=["okrs"], + ) + + +def make_group(pkg, name, path=None): + if path is None: + path = "schema.yml" + + return Group( + name=name, + resource_type=NodeType.Group, + path=path, + package_name=pkg, + original_file_path=path, + unique_id=f"group.{pkg}.{name}", + owner="email@gmail.com", + ) + + +def make_semantic_model( + pkg: str, + name: str, + model, + path=None, +): + if path is None: + path = "schema.yml" + + return SemanticModel( + name=name, + resource_type=NodeType.SemanticModel, + model=model, + node_relation=NodeRelation( + alias=model.alias, + schema_name="dbt", + relation_name=model.name, + ), + package_name=pkg, + path=path, + description="Customer entity", + primary_entity="customer", + unique_id=f"semantic_model.{pkg}.{name}", + original_file_path=path, + fqn=[pkg, "semantic_models", name], + ) + + +def make_saved_query(pkg: str, name: str, metric: str, path=None): + if path is None: + path = "schema.yml" + + return SavedQuery( + name=name, + resource_type=NodeType.SavedQuery, + package_name=pkg, + path=path, + description="Test Saved Query", + query_params=QueryParams( + metrics=[metric], + group_by=[], + where=None, + ), + exports=[], + unique_id=f"saved_query.{pkg}.{name}", + original_file_path=path, + fqn=[pkg, "saved_queries", name], + ) + + +@pytest.fixture +def macro_test_unique(): + return make_macro( + "dbt", "test_unique", "blablabla", depends_on_macros=["macro.dbt.default__test_unique"] + ) + + +@pytest.fixture +def macro_default_test_unique(): + return make_macro("dbt", "default__test_unique", "blablabla") + + +@pytest.fixture +def macro_test_not_null(): + return make_macro( + "dbt", "test_not_null", "blablabla", depends_on_macros=["macro.dbt.default__test_not_null"] + ) + + +@pytest.fixture +def macro_default_test_not_null(): + return make_macro("dbt", "default__test_not_null", "blabla") + + +@pytest.fixture +def seed(): + return make_seed("pkg", "seed") + + +@pytest.fixture +def source(): + return make_source("pkg", "raw", "seed", identifier="seed") + + +@pytest.fixture +def ephemeral_model(source): + return make_model( + "pkg", + "ephemeral_model", + 'select * from {{ source("raw", "seed") }}', + config_kwargs={"materialized": "ephemeral"}, + sources=[source], + ) + + +@pytest.fixture +def view_model(ephemeral_model): + return make_model( + "pkg", + "view_model", + 'select * from {{ ref("ephemeral_model") }}', + config_kwargs={"materialized": "view"}, + refs=[ephemeral_model], + tags=["uses_ephemeral"], + ) + + +@pytest.fixture +def table_model(ephemeral_model): + return make_model( + "pkg", + "table_model", + 'select * from {{ ref("ephemeral_model") }}', + config_kwargs={ + "materialized": "table", + "meta": { + # Other properties to test in test_select_config_meta + "string_property": "some_string", + "truthy_bool_property": True, + "falsy_bool_property": False, + "list_property": ["some_value", True, False], + }, + }, + refs=[ephemeral_model], + tags=["uses_ephemeral"], + path="subdirectory/table_model.sql", + ) + + +@pytest.fixture +def table_model_py(seed): + return make_model( + "pkg", + "table_model_py", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + tags=[], + path="subdirectory/table_model.py", + ) + + +@pytest.fixture +def table_model_csv(seed): + return make_model( + "pkg", + "table_model_csv", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + tags=[], + path="subdirectory/table_model.csv", + ) + + +@pytest.fixture +def ext_source(): + return make_source( + "ext", + "ext_raw", + "ext_source", + ) + + +@pytest.fixture +def ext_source_2(): + return make_source( + "ext", + "ext_raw", + "ext_source_2", + ) + + +@pytest.fixture +def ext_source_other(): + return make_source( + "ext", + "raw", + "ext_source", + ) + + +@pytest.fixture +def ext_source_other_2(): + return make_source( + "ext", + "raw", + "ext_source_2", + ) + + +@pytest.fixture +def ext_model(ext_source): + return make_model( + "ext", + "ext_model", + 'select * from {{ source("ext_raw", "ext_source") }}', + sources=[ext_source], + ) + + +@pytest.fixture +def union_model(seed, ext_source): + return make_model( + "pkg", + "union_model", + 'select * from {{ ref("seed") }} union all select * from {{ source("ext_raw", "ext_source") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[ext_source], + fqn_extras=["unions"], + path="subdirectory/union_model.sql", + tags=["unions"], + ) + + +@pytest.fixture +def versioned_model_v1(seed): + return make_model( + "pkg", + "versioned_model", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[], + path="subdirectory/versioned_model_v1.sql", + version=1, + latest_version=2, + ) + + +@pytest.fixture +def versioned_model_v2(seed): + return make_model( + "pkg", + "versioned_model", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[], + path="subdirectory/versioned_model_v2.sql", + version=2, + latest_version=2, + ) + + +@pytest.fixture +def versioned_model_v3(seed): + return make_model( + "pkg", + "versioned_model", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[], + path="subdirectory/versioned_model_v3.sql", + version="3", + latest_version=2, + ) + + +@pytest.fixture +def versioned_model_v12_string(seed): + return make_model( + "pkg", + "versioned_model", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[], + path="subdirectory/versioned_model_v12.sql", + version="12", + latest_version=2, + ) + + +@pytest.fixture +def versioned_model_v4_nested_dir(seed): + return make_model( + "pkg", + "versioned_model", + 'select * from {{ ref("seed") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[], + path="subdirectory/nested_dir/versioned_model_v3.sql", + version="4", + latest_version=2, + fqn_extras=["nested_dir"], + ) + + +@pytest.fixture +def table_id_unique(table_model): + return make_unique_test("pkg", table_model, "id") + + +@pytest.fixture +def table_id_not_null(table_model): + return make_not_null_test("pkg", table_model, "id") + + +@pytest.fixture +def view_id_unique(view_model): + return make_unique_test("pkg", view_model, "id") + + +@pytest.fixture +def ext_source_id_unique(ext_source): + return make_unique_test("ext", ext_source, "id") + + +@pytest.fixture +def view_test_nothing(view_model): + return make_singular_test( + "pkg", + "view_test_nothing", + 'select * from {{ ref("view_model") }} limit 0', + refs=[view_model], + ) + + +@pytest.fixture +def unit_test_table_model(table_model): + return make_unit_test( + "pkg", + "unit_test_table_model", + table_model, + ) + + +# Support dots as namespace separators +@pytest.fixture +def namespaced_seed(): + return make_seed("pkg", "mynamespace.seed") + + +@pytest.fixture +def namespace_model(source): + return make_model( + "pkg", + "mynamespace.ephemeral_model", + 'select * from {{ source("raw", "seed") }}', + config_kwargs={"materialized": "ephemeral"}, + sources=[source], + ) + + +@pytest.fixture +def namespaced_union_model(seed, ext_source): + return make_model( + "pkg", + "mynamespace.union_model", + 'select * from {{ ref("mynamespace.seed") }} union all select * from {{ ref("mynamespace.ephemeral_model") }}', + config_kwargs={"materialized": "table"}, + refs=[seed], + sources=[ext_source], + fqn_extras=["unions"], + path="subdirectory/union_model.sql", + tags=["unions"], + ) + + +@pytest.fixture +def metric() -> Metric: + return Metric( + name="my_metric", + resource_type=NodeType.Metric, + type=MetricType.SIMPLE, + type_params=MetricTypeParams(measure=MetricInputMeasure(name="a_measure")), + fqn=["test", "metrics", "myq_metric"], + unique_id="metric.test.my_metric", + package_name="test", + path="models/metric.yml", + original_file_path="models/metric.yml", + description="", + meta={}, + tags=[], + label="test_label", + ) + + +@pytest.fixture +def saved_query() -> SavedQuery: + pkg = "test" + name = "test_saved_query" + path = "test_path" + return SavedQuery( + name=name, + resource_type=NodeType.SavedQuery, + package_name=pkg, + path=path, + description="Test Saved Query", + query_params=QueryParams( + metrics=["my_metric"], + group_by=[], + where=None, + ), + exports=[], + unique_id=f"saved_query.{pkg}.{name}", + original_file_path=path, + fqn=[pkg, "saved_queries", name], + ) + + +@pytest.fixture +def semantic_model(table_model) -> SemanticModel: + return make_semantic_model("test", "test_semantic_model", model=table_model) + + +@pytest.fixture +def metricflow_time_spine_model() -> ModelNode: + return ModelNode( + name="metricflow_time_spine", + database="dbt", + schema="analytics", + alias="events", + resource_type=NodeType.Model, + unique_id="model.test.metricflow_time_spine", + fqn=["snowplow", "events"], + package_name="snowplow", + refs=[], + sources=[], + metrics=[], + depends_on=DependsOn(), + config=ModelConfig(), + tags=[], + path="events.sql", + original_file_path="events.sql", + meta={}, + language="sql", + raw_code="does not matter", + checksum=FileHash.empty(), + relation_name="events", + ) + + +@pytest.fixture +def nodes( + seed, + ephemeral_model, + view_model, + table_model, + table_model_py, + table_model_csv, + union_model, + versioned_model_v1, + versioned_model_v2, + versioned_model_v3, + versioned_model_v4_nested_dir, + versioned_model_v12_string, + ext_model, + table_id_unique, + table_id_not_null, + view_id_unique, + ext_source_id_unique, + view_test_nothing, + namespaced_seed, + namespace_model, + namespaced_union_model, +) -> list: + return [ + seed, + ephemeral_model, + view_model, + table_model, + table_model_py, + table_model_csv, + union_model, + versioned_model_v1, + versioned_model_v2, + versioned_model_v3, + versioned_model_v4_nested_dir, + versioned_model_v12_string, + ext_model, + table_id_unique, + table_id_not_null, + view_id_unique, + ext_source_id_unique, + view_test_nothing, + namespaced_seed, + namespace_model, + namespaced_union_model, + ] + + +@pytest.fixture +def sources( + source, + ext_source, + ext_source_2, + ext_source_other, + ext_source_other_2, +) -> list: + return [source, ext_source, ext_source_2, ext_source_other, ext_source_other_2] + + +@pytest.fixture +def macros( + macro_test_unique, + macro_default_test_unique, + macro_test_not_null, + macro_default_test_not_null, +) -> list: + return [ + macro_test_unique, + macro_default_test_unique, + macro_test_not_null, + macro_default_test_not_null, + ] + + +@pytest.fixture +def unit_tests(unit_test_table_model) -> list: + return [unit_test_table_model] + + +@pytest.fixture +def metrics() -> list: + return [] + + +@pytest.fixture +def semantic_models() -> list: + return [] + + +@pytest.fixture +def manifest( + metric, + semantic_model, + nodes, + sources, + macros, + unit_tests, + metrics, + semantic_models, +) -> Manifest: + manifest = Manifest( + nodes={n.unique_id: n for n in nodes}, + sources={s.unique_id: s for s in sources}, + macros={m.unique_id: m for m in macros}, + unit_tests={t.unique_id: t for t in unit_tests}, + semantic_models={s.unique_id: s for s in semantic_models}, + docs={}, + files={}, + exposures={}, + metrics={m.unique_id: m for m in metrics}, + disabled={}, + selectors={}, + groups={}, + metadata=ManifestMetadata(adapter_type="postgres"), + ) + return manifest From 86b349f81218550f3813e82ede5bb912e2a9dcfc Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 16 Apr 2024 20:17:40 -0400 Subject: [PATCH 07/17] Support using sql in unit testing fixtures (#9873) --- .../unreleased/Features-20240408-094132.yaml | 6 + .../resources/v1/unit_test_definition.py | 1 + core/dbt/contracts/graph/model_config.py | 1 + core/dbt/contracts/graph/nodes.py | 2 +- core/dbt/parser/fixtures.py | 7 +- core/dbt/parser/read_files.py | 6 +- core/dbt/parser/unit_tests.py | 51 +++- .../unit_testing/test_sql_format.py | 245 ++++++++++++++++++ 8 files changed, 302 insertions(+), 17 deletions(-) create mode 100644 .changes/unreleased/Features-20240408-094132.yaml create mode 100644 tests/functional/unit_testing/test_sql_format.py diff --git a/.changes/unreleased/Features-20240408-094132.yaml b/.changes/unreleased/Features-20240408-094132.yaml new file mode 100644 index 00000000000..0b7a251e926 --- /dev/null +++ b/.changes/unreleased/Features-20240408-094132.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support SQL in unit testing fixtures +time: 2024-04-08T09:41:32.15936-04:00 +custom: + Author: gshank + Issue: "9405" diff --git a/core/dbt/artifacts/resources/v1/unit_test_definition.py b/core/dbt/artifacts/resources/v1/unit_test_definition.py index 7ef10f52cdf..fc265fa36b9 100644 --- a/core/dbt/artifacts/resources/v1/unit_test_definition.py +++ b/core/dbt/artifacts/resources/v1/unit_test_definition.py @@ -30,6 +30,7 @@ class UnitTestConfig(BaseConfig): class UnitTestFormat(StrEnum): CSV = "csv" Dict = "dict" + SQL = "sql" @dataclass diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 18765dc5eaa..b45c313327c 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -36,6 +36,7 @@ def insensitive_patterns(*patterns: str): @dataclass class UnitTestNodeConfig(NodeConfig): expected_rows: List[Dict[str, Any]] = field(default_factory=list) + expected_sql: Optional[str] = None @dataclass diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 134c272db23..e1f409ff1de 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -991,7 +991,7 @@ def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool: @dataclass class UnitTestFileFixture(BaseNode): resource_type: Literal[NodeType.Fixture] - rows: Optional[List[Dict[str, Any]]] = None + rows: Optional[Union[List[Dict[str, Any]], str]] = None # ==================================== diff --git a/core/dbt/parser/fixtures.py b/core/dbt/parser/fixtures.py index f12cc6f272a..b3002725674 100644 --- a/core/dbt/parser/fixtures.py +++ b/core/dbt/parser/fixtures.py @@ -26,6 +26,11 @@ def parse_file(self, file_block: FileBlock): assert isinstance(file_block.file, FixtureSourceFile) unique_id = self.generate_unique_id(file_block.name) + if file_block.file.path.relative_path.endswith(".sql"): + rows = file_block.file.contents # type: ignore + else: # endswith('.csv') + rows = self.get_rows(file_block.file.contents) # type: ignore + fixture = UnitTestFileFixture( name=file_block.name, path=file_block.file.path.relative_path, @@ -33,7 +38,7 @@ def parse_file(self, file_block: FileBlock): package_name=self.project.project_name, unique_id=unique_id, resource_type=NodeType.Fixture, - rows=self.get_rows(file_block.file.contents), + rows=rows, ) self.manifest.add_fixture(file_block.file, fixture) diff --git a/core/dbt/parser/read_files.py b/core/dbt/parser/read_files.py index a44bd2fbb22..314a2a0fdd1 100644 --- a/core/dbt/parser/read_files.py +++ b/core/dbt/parser/read_files.py @@ -145,11 +145,11 @@ def get_source_files(project, paths, extension, parse_file_type, saved_files, ig if parse_file_type == ParseFileType.Seed: fb_list.append(load_seed_source_file(fp, project.project_name)) # singular tests live in /tests but only generic tests live - # in /tests/generic so we want to skip those + # in /tests/generic and fixtures in /tests/fixture so we want to skip those else: if parse_file_type == ParseFileType.SingularTest: path = pathlib.Path(fp.relative_path) - if path.parts[0] == "generic": + if path.parts[0] in ["generic", "fixtures"]: continue file = load_source_file(fp, parse_file_type, project.project_name, saved_files) # only append the list if it has contents. added to fix #3568 @@ -431,7 +431,7 @@ def get_file_types_for_project(project): }, ParseFileType.Fixture: { "paths": project.fixture_paths, - "extensions": [".csv"], + "extensions": [".csv", ".sql"], "parser": "FixtureParser", }, } diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 763efab44aa..0abadca5cf9 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -68,6 +68,15 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): name = test_case.name if tested_node.is_versioned: name = name + f"_v{tested_node.version}" + expected_sql: Optional[str] = None + if test_case.expect.format == UnitTestFormat.SQL: + expected_rows: List[Dict[str, Any]] = [] + expected_sql = test_case.expect.rows # type: ignore + else: + assert isinstance(test_case.expect.rows, List) + expected_rows = deepcopy(test_case.expect.rows) + + assert isinstance(expected_rows, List) unit_test_node = UnitTestNode( name=name, resource_type=NodeType.Unit, @@ -76,8 +85,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): original_file_path=test_case.original_file_path, unique_id=test_case.unique_id, config=UnitTestNodeConfig( - materialized="unit", - expected_rows=deepcopy(test_case.expect.rows), # type:ignore + materialized="unit", expected_rows=expected_rows, expected_sql=expected_sql ), raw_code=tested_node.raw_code, database=tested_node.database, @@ -132,7 +140,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): "schema": original_input_node.schema, "fqn": original_input_node.fqn, "checksum": FileHash.empty(), - "raw_code": self._build_fixture_raw_code(given.rows, None), + "raw_code": self._build_fixture_raw_code(given.rows, None, given.format), "package_name": original_input_node.package_name, "unique_id": f"model.{original_input_node.package_name}.{input_name}", "name": input_name, @@ -172,12 +180,15 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): # Add unique ids of input_nodes to depends_on unit_test_node.depends_on.nodes.append(input_node.unique_id) - def _build_fixture_raw_code(self, rows, column_name_to_data_types) -> str: + def _build_fixture_raw_code(self, rows, column_name_to_data_types, fixture_format) -> str: # We're not currently using column_name_to_data_types, but leaving here for # possible future use. - return ("{{{{ get_fixture_sql({rows}, {column_name_to_data_types}) }}}}").format( - rows=rows, column_name_to_data_types=column_name_to_data_types - ) + if fixture_format == UnitTestFormat.SQL: + return rows + else: + return ("{{{{ get_fixture_sql({rows}, {column_name_to_data_types}) }}}}").format( + rows=rows, column_name_to_data_types=column_name_to_data_types + ) def _get_original_input_node(self, input: str, tested_node: ModelNode, test_case_name: str): """ @@ -352,13 +363,29 @@ def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture ) if ut_fixture.fixture: - # find fixture file object and store unit_test_definition unique_id - fixture = self._get_fixture(ut_fixture.fixture, self.project.project_name) - fixture_source_file = self.manifest.files[fixture.file_id] - fixture_source_file.unit_tests.append(unit_test_definition.unique_id) - ut_fixture.rows = fixture.rows + ut_fixture.rows = self.get_fixture_file_rows( + ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id + ) else: ut_fixture.rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) + elif ut_fixture.format == UnitTestFormat.SQL: + if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)): + raise ParsingError( + f"Unit test {unit_test_definition.name} has {fixture_type} rows or fixtures " + f"which do not match format {ut_fixture.format}. Expected string." + ) + + if ut_fixture.fixture: + ut_fixture.rows = self.get_fixture_file_rows( + ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id + ) + + def get_fixture_file_rows(self, fixture_name, project_name, utdef_unique_id): + # find fixture file object and store unit_test_definition unique_id + fixture = self._get_fixture(fixture_name, project_name) + fixture_source_file = self.manifest.files[fixture.file_id] + fixture_source_file.unit_tests.append(utdef_unique_id) + return fixture.rows def _convert_csv_to_list_of_dicts(self, csv_string: str) -> List[Dict[str, Any]]: dummy_file = StringIO(csv_string) diff --git a/tests/functional/unit_testing/test_sql_format.py b/tests/functional/unit_testing/test_sql_format.py new file mode 100644 index 00000000000..6b5af93e1ba --- /dev/null +++ b/tests/functional/unit_testing/test_sql_format.py @@ -0,0 +1,245 @@ +import pytest +from dbt.tests.util import run_dbt + +wizards_csv = """id,w_name,email,email_tld,phone,world +1,Albus Dumbledore,a.dumbledore@gmail.com,gmail.com,813-456-9087,1 +2,Gandalf,gandy811@yahoo.com,yahoo.com,551-329-8367,2 +3,Winifred Sanderson,winnie@hocuspocus.com,hocuspocus.com,,6 +4,Marnie Piper,cromwellwitch@gmail.com,gmail.com,,5 +5,Grace Goheen,grace.goheen@dbtlabs.com,dbtlabs.com,,3 +6,Glinda,glinda_good@hotmail.com,hotmail.com,912-458-3289,4 +""" + +top_level_email_domains_csv = """tld +gmail.com +yahoo.com +hocuspocus.com +dbtlabs.com +hotmail.com +""" + +worlds_csv = """id,name +1,The Wizarding World +2,Middle-earth +3,dbt Labs +4,Oz +5,Halloweentown +6,Salem +""" + +stg_wizards_sql = """ +select + id as wizard_id, + w_name as wizard_name, + email, + email_tld as email_top_level_domain, + phone as phone_number, + world as world_id +from {{ ref('wizards') }} +""" + +stg_worlds_sql = """ +select + id as world_id, + name as world_name +from {{ ref('worlds') }} +""" + +dim_wizards_sql = """ +with wizards as ( + + select * from {{ ref('stg_wizards') }} + +), + +worlds as ( + + select * from {{ ref('stg_worlds') }} + +), + +accepted_email_domains as ( + + select * from {{ ref('top_level_email_domains') }} + +), + +check_valid_emails as ( + + select + wizards.wizard_id, + wizards.wizard_name, + wizards.email, + wizards.phone_number, + wizards.world_id, + + coalesce ( + wizards.email ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$' + = true + and accepted_email_domains.tld is not null, + false) as is_valid_email_address + + from wizards + left join accepted_email_domains + on wizards.email_top_level_domain = lower(accepted_email_domains.tld) + +) + +select + check_valid_emails.wizard_id, + check_valid_emails.wizard_name, + check_valid_emails.email, + check_valid_emails.is_valid_email_address, + check_valid_emails.phone_number, + worlds.world_name +from check_valid_emails +left join worlds + on check_valid_emails.world_id = worlds.world_id +""" + +orig_schema_yml = """ +unit_tests: + - name: test_valid_email_address + model: dim_wizards + given: + - input: ref('stg_wizards') + rows: + - {email: cool@example.com, email_top_level_domain: example.com} + - {email: cool@unknown.com, email_top_level_domain: unknown.com} + - {email: badgmail.com, email_top_level_domain: gmail.com} + - {email: missingdot@gmailcom, email_top_level_domain: gmail.com} + - input: ref('top_level_email_domains') + rows: + - {tld: example.com} + - {tld: gmail.com} + - input: ref('stg_worlds') + rows: [] + expect: + rows: + - {email: cool@example.com, is_valid_email_address: true} + - {email: cool@unknown.com, is_valid_email_address: false} + - {email: badgmail.com, is_valid_email_address: false} + - {email: missingdot@gmailcom, is_valid_email_address: false} +""" + +schema_yml = """ +unit_tests: + - name: test_valid_email_address + model: dim_wizards + given: + - input: ref('stg_wizards') + format: sql + rows: | + select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, 'example.com' as email_top_level_domain, '123' as phone_number, 1 as world_id union all + select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, 'unknown.com' as email_top_level_domain, '456' as phone_number, 2 as world_id union all + select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, 'gmail.com' as email_top_level_domain, '789' as phone_number, 3 as world_id union all + select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, 'gmail.com' as email_top_level_domain, '102' as phone_number, 4 as world_id + - input: ref('top_level_email_domains') + format: sql + rows: | + select 'example.com' as tld union all + select 'gmail.com' as tld + - input: ref('stg_worlds') + rows: [] + expect: + format: sql + rows: | + select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, true as is_valid_email_address, '123' as phone_number, null as world_name union all + select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, false as is_valid_email_address, '456' as phone_number, null as world_name union all + select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, false as is_valid_email_address, '789' as phone_number, null as world_name union all + select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, false as is_valid_email_address, '102' as phone_number, null as world_name +""" + + +class TestSQLFormat: + @pytest.fixture(scope="class") + def seeds(self): + return { + "wizards.csv": wizards_csv, + "top_level_email_domains.csv": top_level_email_domains_csv, + "worlds.csv": worlds_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "stg_wizards.sql": stg_wizards_sql, + "stg_worlds.sql": stg_worlds_sql, + "dim_wizards.sql": dim_wizards_sql, + "schema.yml": schema_yml, + } + + def test_sql_format(self, project): + results = run_dbt(["build"]) + assert len(results) == 7 + + +stg_wizards_fixture_sql = """ + select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, 'example.com' as email_top_level_domain, '123' as phone_number, 1 as world_id union all + select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, 'unknown.com' as email_top_level_domain, '456' as phone_number, 2 as world_id union all + select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, 'gmail.com' as email_top_level_domain, '789' as phone_number, 3 as world_id union all + select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, 'gmail.com' as email_top_level_domain, '102' as phone_number, 4 as world_id +""" + +top_level_email_domains_fixture_sql = """ + select 'example.com' as tld union all + select 'gmail.com' as tld +""" + +test_valid_email_address_fixture_sql = """ + select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, true as is_valid_email_address, '123' as phone_number, null as world_name union all + select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, false as is_valid_email_address, '456' as phone_number, null as world_name union all + select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, false as is_valid_email_address, '789' as phone_number, null as world_name union all + select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, false as is_valid_email_address, '102' as phone_number, null as world_name +""" + +fixture_schema_yml = """ +unit_tests: + - name: test_valid_email_address + model: dim_wizards + given: + - input: ref('stg_wizards') + format: sql + fixture: stg_wizards_fixture + - input: ref('top_level_email_domains') + format: sql + fixture: top_level_email_domains_fixture + - input: ref('stg_worlds') + rows: [] + expect: + format: sql + fixture: test_valid_email_address_fixture +""" + + +class TestSQLFormatFixtures: + @pytest.fixture(scope="class") + def tests(self): + return { + "fixtures": { + "test_valid_email_address_fixture.sql": test_valid_email_address_fixture_sql, + "top_level_email_domains_fixture.sql": top_level_email_domains_fixture_sql, + "stg_wizards_fixture.sql": stg_wizards_fixture_sql, + } + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "wizards.csv": wizards_csv, + "top_level_email_domains.csv": top_level_email_domains_csv, + "worlds.csv": worlds_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "stg_wizards.sql": stg_wizards_sql, + "stg_worlds.sql": stg_worlds_sql, + "dim_wizards.sql": dim_wizards_sql, + "schema.yml": fixture_schema_yml, + } + + def test_sql_format_fixtures(self, project): + results = run_dbt(["build"]) + assert len(results) == 7 From 5cb127999cc3a7a230c8725b04bb8ee9075cba8d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 17 Apr 2024 10:22:12 -0400 Subject: [PATCH 08/17] add test for unit test that depends on ephemeral model (#9929) --- .../unit_testing/test_ut_ephemeral.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/functional/unit_testing/test_ut_ephemeral.py diff --git a/tests/functional/unit_testing/test_ut_ephemeral.py b/tests/functional/unit_testing/test_ut_ephemeral.py new file mode 100644 index 00000000000..2898633ec40 --- /dev/null +++ b/tests/functional/unit_testing/test_ut_ephemeral.py @@ -0,0 +1,84 @@ +import pytest +from dbt.tests.util import run_dbt, write_file +from dbt.contracts.results import RunStatus, TestStatus + + +ephemeral_model_sql = """ +{{ config(materialized="ephemeral") }} +select 1 as id, 'Emily' as first_name +""" + +nested_ephemeral_model_sql = """ +{{ config(materialized="ephemeral") }} +select * from {{ ref('ephemeral_model') }} +""" + +customers_sql = """ +select * from {{ ref('nested_ephemeral_model') }} +""" + +test_sql_format_yml = """ +unit_tests: + - name: test_customers + model: customers + given: + - input: ref('nested_ephemeral_model') + format: sql + rows: | + select 1 as id, 'Emily' as first_name + expect: + rows: + - {id: 1, first_name: Emily} +""" + +failing_test_sql_format_yml = """ + - name: fail_test_customers + model: customers + given: + - input: ref('nested_ephemeral_model') + format: sql + rows: | + select 1 as id, 'Emily' as first_name + expect: + rows: + - {id: 1, first_name: Joan} +""" + + +class TestUnitTestEphemeralInput: + @pytest.fixture(scope="class") + def models(self): + return { + "customers.sql": customers_sql, + "ephemeral_model.sql": ephemeral_model_sql, + "nested_ephemeral_model.sql": nested_ephemeral_model_sql, + "tests.yml": test_sql_format_yml, + } + + def test_ephemeral_input(self, project): + results = run_dbt(["run"]) + len(results) == 1 + + results = run_dbt(["test", "--select", "test_type:unit"]) + assert len(results) == 1 + + results = run_dbt(["build"]) + assert len(results) == 2 + result_unique_ids = [result.node.unique_id for result in results] + assert len(result_unique_ids) == 2 + assert "unit_test.test.customers.test_customers" in result_unique_ids + + # write failing unit test + write_file( + test_sql_format_yml + failing_test_sql_format_yml, + project.project_root, + "models", + "tests.yml", + ) + results = run_dbt(["build"], expect_pass=False) + for result in results: + if result.node.unique_id == "model.test.customers": + assert result.status == RunStatus.Skipped + elif result.node.unique_id == "unit_test.test.customers.fail_test_customers": + assert result.status == TestStatus.Fail + assert len(results) == 3 From fe28d9e115f9a31cedcc641f67fcd4d229ad0027 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 17 Apr 2024 10:37:49 -0400 Subject: [PATCH 09/17] Revert "revert python version for docker images" (#9687) --- .changes/unreleased/Dependencies-20240227-142138.yaml | 6 ++++++ docker/Dockerfile | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Dependencies-20240227-142138.yaml diff --git a/.changes/unreleased/Dependencies-20240227-142138.yaml b/.changes/unreleased/Dependencies-20240227-142138.yaml new file mode 100644 index 00000000000..8c68091e07e --- /dev/null +++ b/.changes/unreleased/Dependencies-20240227-142138.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker +time: 2024-02-27T14:21:38.394757-05:00 +custom: + Author: michelleark + PR: "9687" diff --git a/docker/Dockerfile b/docker/Dockerfile index 10e63d3ec27..5b07514d76b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,4 @@ -# this image gets published to GHCR for production use -ARG py_version=3.10.7 +ARG py_version=3.11.2 FROM python:$py_version-slim-bullseye as base From 668fe78e2dc39a15a113266dedab0ae0b62b6aed Mon Sep 17 00:00:00 2001 From: Damian Owsianny <113946744+damian3031@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:00:48 +0200 Subject: [PATCH 10/17] Fix query comments test (#9861) --- .changes/unreleased/Fixes-20240408-130646.yaml | 6 ++++++ tests/functional/adapter/query_comment/fixtures.py | 1 - .../functional/adapter/query_comment/test_query_comment.py | 2 -- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240408-130646.yaml diff --git a/.changes/unreleased/Fixes-20240408-130646.yaml b/.changes/unreleased/Fixes-20240408-130646.yaml new file mode 100644 index 00000000000..9aeaa94a27c --- /dev/null +++ b/.changes/unreleased/Fixes-20240408-130646.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fixed query comments test +time: 2024-04-08T13:06:46.648144+02:00 +custom: + Author: damian3031 + Issue: "9860" diff --git a/tests/functional/adapter/query_comment/fixtures.py b/tests/functional/adapter/query_comment/fixtures.py index d8848dc089e..ccaf329209c 100644 --- a/tests/functional/adapter/query_comment/fixtures.py +++ b/tests/functional/adapter/query_comment/fixtures.py @@ -10,7 +10,6 @@ {%- set comment_dict = dict( app='dbt++', macro_version='0.1.0', - dbt_version=dbt_version, message='blah: '~ message) -%} {{ return(comment_dict) }} {%- endmacro -%} diff --git a/tests/functional/adapter/query_comment/test_query_comment.py b/tests/functional/adapter/query_comment/test_query_comment.py index 18d66ffda7d..5651e54b39b 100644 --- a/tests/functional/adapter/query_comment/test_query_comment.py +++ b/tests/functional/adapter/query_comment/test_query_comment.py @@ -1,7 +1,6 @@ import pytest import json from dbt.exceptions import DbtRuntimeError -from dbt.version import __version__ as dbt_version from dbt.tests.util import run_dbt_and_capture from tests.functional.adapter.query_comment.fixtures import MACROS__MACRO_SQL, MODELS__X_SQL @@ -59,7 +58,6 @@ def test_matches_comment(self, project) -> bool: logs = self.run_get_json() expected_dct = { "app": "dbt++", - "dbt_version": dbt_version, "macro_version": "0.1.0", "message": f"blah: {project.adapter.config.target_name}", } From 2edd5b3335ddea4ca5e7dd8aaa3bfe3a5d30499a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 17 Apr 2024 14:28:27 -0500 Subject: [PATCH 11/17] fix change kind (#9964) --- .changes/unreleased/Dependencies-20240415-202426.yaml | 6 ------ .changes/unreleased/Security-20240417-141316.yaml | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 .changes/unreleased/Dependencies-20240415-202426.yaml create mode 100644 .changes/unreleased/Security-20240417-141316.yaml diff --git a/.changes/unreleased/Dependencies-20240415-202426.yaml b/.changes/unreleased/Dependencies-20240415-202426.yaml deleted file mode 100644 index d009f63a154..00000000000 --- a/.changes/unreleased/Dependencies-20240415-202426.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Bump sqlparse to >=0.5.0, <0.6.0 -time: 2024-04-15T20:24:26.768707-05:00 -custom: - Author: emmyoop - Issue: "9951" diff --git a/.changes/unreleased/Security-20240417-141316.yaml b/.changes/unreleased/Security-20240417-141316.yaml new file mode 100644 index 00000000000..6611cafb443 --- /dev/null +++ b/.changes/unreleased/Security-20240417-141316.yaml @@ -0,0 +1,6 @@ +kind: Security +body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg +time: 2024-04-17T14:13:16.896353-05:00 +custom: + Author: emmoop + Issue: "9951" From f87964ec1c3a2d9fa7158bdf72e194c62b20228b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 18 Apr 2024 09:12:00 -0500 Subject: [PATCH 12/17] updated unit test paths (#9963) --- tests/unit/{test_freshness_task.py => task/test_freshness.py} | 0 tests/unit/{test_graph_runnable_task.py => task/test_runnable.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/unit/{test_freshness_task.py => task/test_freshness.py} (100%) rename tests/unit/{test_graph_runnable_task.py => task/test_runnable.py} (100%) diff --git a/tests/unit/test_freshness_task.py b/tests/unit/task/test_freshness.py similarity index 100% rename from tests/unit/test_freshness_task.py rename to tests/unit/task/test_freshness.py diff --git a/tests/unit/test_graph_runnable_task.py b/tests/unit/task/test_runnable.py similarity index 100% rename from tests/unit/test_graph_runnable_task.py rename to tests/unit/task/test_runnable.py From 61727ab5b626e54a6e27793eea8771f6ecec28a9 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 18 Apr 2024 09:35:24 -0500 Subject: [PATCH 13/17] fix changelogs (#9975) --- .changes/unreleased/Dependencies-20240227-142138.yaml | 2 +- .changes/unreleased/Dependencies-20240410-183321.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/unreleased/Dependencies-20240227-142138.yaml b/.changes/unreleased/Dependencies-20240227-142138.yaml index 8c68091e07e..0b20244d773 100644 --- a/.changes/unreleased/Dependencies-20240227-142138.yaml +++ b/.changes/unreleased/Dependencies-20240227-142138.yaml @@ -3,4 +3,4 @@ body: Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker time: 2024-02-27T14:21:38.394757-05:00 custom: Author: michelleark - PR: "9687" + Issue: "9687" diff --git a/.changes/unreleased/Dependencies-20240410-183321.yaml b/.changes/unreleased/Dependencies-20240410-183321.yaml index 4cdcd6af835..7fb86e98c3b 100644 --- a/.changes/unreleased/Dependencies-20240410-183321.yaml +++ b/.changes/unreleased/Dependencies-20240410-183321.yaml @@ -3,4 +3,4 @@ body: "Bump black from 23.3.0 to >=24.3.0,<25.0" time: 2024-04-10T18:33:21.00000Z custom: Author: dependabot[bot] - PR: 8074 + Issue: 8074 From f5f9591d09550c4cfd425c2cee702c9b0ade5e33 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 18 Apr 2024 10:44:28 -0400 Subject: [PATCH 14/17] [Robust Testing] Move tests to tests/unit/context (#9966) --- core/dbt/cli/requires.py | 2 +- core/dbt/context/manifest.py | 10 -- core/dbt/context/query_header.py | 13 ++ core/dbt/parser/manifest.py | 2 +- tests/unit/{ => context}/test_context.py | 170 ++++++++++-------- tests/unit/{ => context}/test_providers.py | 0 .../test_query_header.py} | 49 ++--- 7 files changed, 141 insertions(+), 105 deletions(-) create mode 100644 core/dbt/context/query_header.py rename tests/unit/{ => context}/test_context.py (81%) rename tests/unit/{ => context}/test_providers.py (100%) rename tests/unit/{test_query_headers.py => context/test_query_header.py} (50%) diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index ccd5ffc7150..75c81ebd7e1 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -15,7 +15,7 @@ from dbt.cli.flags import Flags from dbt.config import RuntimeConfig from dbt.config.runtime import load_project, load_profile, UnsetProfile -from dbt.context.manifest import generate_query_header_context +from dbt.context.query_header import generate_query_header_context from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import ( diff --git a/core/dbt/context/manifest.py b/core/dbt/context/manifest.py index d55d3ad0f21..0d95fd3b95f 100644 --- a/core/dbt/context/manifest.py +++ b/core/dbt/context/manifest.py @@ -71,13 +71,3 @@ def to_dict(self): @contextproperty() def context_macro_stack(self): return self.macro_stack - - -class QueryHeaderContext(ManifestContext): - def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None: - super().__init__(config, manifest, config.project_name) - - -def generate_query_header_context(config: AdapterRequiredConfig, manifest: Manifest): - ctx = QueryHeaderContext(config, manifest) - return ctx.to_dict() diff --git a/core/dbt/context/query_header.py b/core/dbt/context/query_header.py new file mode 100644 index 00000000000..95c5a0b7a8f --- /dev/null +++ b/core/dbt/context/query_header.py @@ -0,0 +1,13 @@ +from dbt.adapters.contracts.connection import AdapterRequiredConfig +from dbt.context.manifest import ManifestContext +from dbt.contracts.graph.manifest import Manifest + + +class QueryHeaderContext(ManifestContext): + def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None: + super().__init__(config, manifest, config.project_name) + + +def generate_query_header_context(config: AdapterRequiredConfig, manifest: Manifest): + ctx = QueryHeaderContext(config, manifest) + return ctx.to_dict() diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index a0cc49faa20..5e406d81d03 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -19,7 +19,7 @@ from itertools import chain import time -from dbt.context.manifest import generate_query_header_context +from dbt.context.query_header import generate_query_header_context from dbt.contracts.graph.semantic_manifest import SemanticManifest from dbt_common.events.base_types import EventLevel from dbt_common.exceptions.base import DbtValidationError diff --git a/tests/unit/test_context.py b/tests/unit/context/test_context.py similarity index 81% rename from tests/unit/test_context.py rename to tests/unit/context/test_context.py index fd23da53c17..6070c24a1b7 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/context/test_context.py @@ -1,4 +1,3 @@ -import unittest import os from typing import Set, Dict, Any from unittest import mock @@ -17,26 +16,28 @@ UnitTestOverrides, ) from dbt.config.project import VarProvider -from dbt.context import base, providers, docs, manifest, macros +from dbt.context import base, providers, docs, macros, query_header from dbt.contracts.files import FileHash from dbt_common.events.functions import reset_metadata_vars +from dbt.flags import set_from_args from dbt.node_types import NodeType import dbt_common.exceptions -from .utils import ( + +from tests.unit.utils import ( config_from_parts_or_dicts, inject_adapter, clear_plugin, ) -from .mock_adapter import adapter_factory -from dbt.flags import set_from_args +from tests.unit.mock_adapter import adapter_factory from argparse import Namespace set_from_args(Namespace(WARN_ERROR=False), None) -class TestVar(unittest.TestCase): - def setUp(self): - self.model = ModelNode( +class TestVar: + @pytest.fixture + def model(self): + return ModelNode( alias="model_one", name="model_one", database="dbt", @@ -70,91 +71,114 @@ def setUp(self): columns={}, checksum=FileHash.from_contents(""), ) - self.context = mock.MagicMock() - self.provider = VarProvider({}) - self.config = mock.MagicMock( - config_version=2, vars=self.provider, cli_vars={}, project_name="root" - ) - def test_var_default_something(self): - self.config.cli_vars = {"foo": "baz"} - var = providers.RuntimeVar(self.context, self.config, self.model) - self.assertEqual(var("foo"), "baz") - self.assertEqual(var("foo", "bar"), "baz") + @pytest.fixture + def context(self): + return mock.MagicMock() + + @pytest.fixture + def provider(self): + return VarProvider({}) + + @pytest.fixture + def config(self, provider): + return mock.MagicMock(config_version=2, vars=provider, cli_vars={}, project_name="root") + + def test_var_default_something(self, model, config, context): + config.cli_vars = {"foo": "baz"} + var = providers.RuntimeVar(context, config, model) + + assert var("foo") == "baz" + assert var("foo", "bar") == "baz" + + def test_var_default_none(self, model, config, context): + config.cli_vars = {"foo": None} + var = providers.RuntimeVar(context, config, model) - def test_var_default_none(self): - self.config.cli_vars = {"foo": None} - var = providers.RuntimeVar(self.context, self.config, self.model) - self.assertEqual(var("foo"), None) - self.assertEqual(var("foo", "bar"), None) + assert var("foo") is None + assert var("foo", "bar") is None - def test_var_not_defined(self): - var = providers.RuntimeVar(self.context, self.config, self.model) + def test_var_not_defined(self, model, config, context): + var = providers.RuntimeVar(self.context, config, model) - self.assertEqual(var("foo", "bar"), "bar") - with self.assertRaises(dbt_common.exceptions.CompilationError): + assert var("foo", "bar") == "bar" + with pytest.raises(dbt_common.exceptions.CompilationError): var("foo") - def test_parser_var_default_something(self): - self.config.cli_vars = {"foo": "baz"} - var = providers.ParseVar(self.context, self.config, self.model) - self.assertEqual(var("foo"), "baz") - self.assertEqual(var("foo", "bar"), "baz") + def test_parser_var_default_something(self, model, config, context): + config.cli_vars = {"foo": "baz"} + var = providers.ParseVar(context, config, model) + assert var("foo") == "baz" + assert var("foo", "bar") == "baz" - def test_parser_var_default_none(self): - self.config.cli_vars = {"foo": None} - var = providers.ParseVar(self.context, self.config, self.model) - self.assertEqual(var("foo"), None) - self.assertEqual(var("foo", "bar"), None) + def test_parser_var_default_none(self, model, config, context): + config.cli_vars = {"foo": None} + var = providers.ParseVar(context, config, model) + assert var("foo") is None + assert var("foo", "bar") is None - def test_parser_var_not_defined(self): + def test_parser_var_not_defined(self, model, config, context): # at parse-time, we should not raise if we encounter a missing var # that way disabled models don't get parse errors - var = providers.ParseVar(self.context, self.config, self.model) + var = providers.ParseVar(context, config, model) - self.assertEqual(var("foo", "bar"), "bar") - self.assertEqual(var("foo"), None) + assert var("foo", "bar") == "bar" + assert var("foo") is None -class TestParseWrapper(unittest.TestCase): - def setUp(self): - self.mock_config = mock.MagicMock() - self.mock_mp_context = mock.MagicMock() +class TestParseWrapper: + @pytest.fixture + def mock_adapter(self): + mock_config = mock.MagicMock() + mock_mp_context = mock.MagicMock() adapter_class = adapter_factory() - self.mock_adapter = adapter_class(self.mock_config, self.mock_mp_context) - self.namespace = mock.MagicMock() - self.wrapper = providers.ParseDatabaseWrapper(self.mock_adapter, self.namespace) - self.responder = self.mock_adapter.responder - - def test_unwrapped_method(self): - self.assertEqual(self.wrapper.quote("test_value"), '"test_value"') - self.responder.quote.assert_called_once_with("test_value") - - def test_wrapped_method(self): - found = self.wrapper.get_relation("database", "schema", "identifier") - self.assertEqual(found, None) - self.responder.get_relation.assert_not_called() - - -class TestRuntimeWrapper(unittest.TestCase): - def setUp(self): - self.mock_config = mock.MagicMock() - self.mock_mp_context = mock.MagicMock() - self.mock_config.quoting = { + return adapter_class(mock_config, mock_mp_context) + + @pytest.fixture + def wrapper(self, mock_adapter): + namespace = mock.MagicMock() + return providers.ParseDatabaseWrapper(mock_adapter, namespace) + + @pytest.fixture + def responder(self, mock_adapter): + return mock_adapter.responder + + def test_unwrapped_method(self, wrapper, responder): + assert wrapper.quote("test_value") == '"test_value"' + responder.quote.assert_called_once_with("test_value") + + def test_wrapped_method(self, wrapper, responder): + found = wrapper.get_relation("database", "schema", "identifier") + assert found is None + responder.get_relation.assert_not_called() + + +class TestRuntimeWrapper: + @pytest.fixture + def mock_adapter(self): + mock_config = mock.MagicMock() + mock_config.quoting = { "database": True, "schema": True, "identifier": True, } + mock_mp_context = mock.MagicMock() adapter_class = adapter_factory() - self.mock_adapter = adapter_class(self.mock_config, self.mock_mp_context) - self.namespace = mock.MagicMock() - self.wrapper = providers.RuntimeDatabaseWrapper(self.mock_adapter, self.namespace) - self.responder = self.mock_adapter.responder + return adapter_class(mock_config, mock_mp_context) + + @pytest.fixture + def wrapper(self, mock_adapter): + namespace = mock.MagicMock() + return providers.RuntimeDatabaseWrapper(mock_adapter, namespace) + + @pytest.fixture + def responder(self, mock_adapter): + return mock_adapter.responder - def test_unwrapped_method(self): + def test_unwrapped_method(self, wrapper, responder): # the 'quote' method isn't wrapped, we should get our expected inputs - self.assertEqual(self.wrapper.quote("test_value"), '"test_value"') - self.responder.quote.assert_called_once_with("test_value") + assert wrapper.quote("test_value") == '"test_value"' + responder.quote.assert_called_once_with("test_value") def assert_has_keys(required_keys: Set[str], maybe_keys: Set[str], ctx: Dict[str, Any]): @@ -417,7 +441,7 @@ def postgres_adapter(config_postgres, get_adapter): def test_query_header_context(config_postgres, manifest_fx): - ctx = manifest.generate_query_header_context( + ctx = query_header.generate_query_header_context( config=config_postgres, manifest=manifest_fx, ) diff --git a/tests/unit/test_providers.py b/tests/unit/context/test_providers.py similarity index 100% rename from tests/unit/test_providers.py rename to tests/unit/context/test_providers.py diff --git a/tests/unit/test_query_headers.py b/tests/unit/context/test_query_header.py similarity index 50% rename from tests/unit/test_query_headers.py rename to tests/unit/context/test_query_header.py index 2be9b59bd4d..aa9e99821a2 100644 --- a/tests/unit/test_query_headers.py +++ b/tests/unit/context/test_query_header.py @@ -1,8 +1,9 @@ +import pytest import re -from unittest import TestCase, mock +from unittest import mock from dbt.adapters.base.query_headers import MacroQueryStringSetter -from dbt.context.manifest import generate_query_header_context +from dbt.context.query_header import generate_query_header_context from tests.unit.utils import config_from_parts_or_dicts from dbt.flags import set_from_args @@ -11,9 +12,10 @@ set_from_args(Namespace(WARN_ERROR=False), None) -class TestQueryHeaders(TestCase): - def setUp(self): - self.profile_cfg = { +class TestQueryHeaderContext: + @pytest.fixture + def profile_cfg(self): + return { "outputs": { "test": { "type": "postgres", @@ -27,33 +29,40 @@ def setUp(self): }, "target": "test", } - self.project_cfg = { + + @pytest.fixture + def project_cfg(self): + return { "name": "query_headers", "version": "0.1", "profile": "test", "config-version": 2, } - self.query = "SELECT 1;" - def test_comment_should_prepend_query_by_default(self): - config = config_from_parts_or_dicts(self.project_cfg, self.profile_cfg) + @pytest.fixture + def query(self): + return "SELECT 1;" + + def test_comment_should_prepend_query_by_default(self, profile_cfg, project_cfg, query): + config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header_context = generate_query_header_context(config, mock.MagicMock(macros={})) query_header = MacroQueryStringSetter(config, query_header_context) - sql = query_header.add(self.query) - self.assertTrue(re.match(f"^\/\*.*\*\/\n{self.query}$", sql)) # noqa: [W605] + sql = query_header.add(query) + assert re.match(f"^\/\*.*\*\/\n{query}$", sql) # noqa: [W605] - def test_append_comment(self): - self.project_cfg.update({"query-comment": {"comment": "executed by dbt", "append": True}}) - config = config_from_parts_or_dicts(self.project_cfg, self.profile_cfg) + def test_append_comment(self, profile_cfg, project_cfg, query): + project_cfg.update({"query-comment": {"comment": "executed by dbt", "append": True}}) + config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header_context = generate_query_header_context(config, mock.MagicMock(macros={})) query_header = MacroQueryStringSetter(config, query_header_context) - sql = query_header.add(self.query) - self.assertEqual(sql, f"{self.query[:-1]}\n/* executed by dbt */;") + sql = query_header.add(query) + + assert sql == f"{query[:-1]}\n/* executed by dbt */;" - def test_disable_query_comment(self): - self.project_cfg.update({"query-comment": ""}) - config = config_from_parts_or_dicts(self.project_cfg, self.profile_cfg) + def test_disable_query_comment(self, profile_cfg, project_cfg, query): + project_cfg.update({"query-comment": ""}) + config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header = MacroQueryStringSetter(config, mock.MagicMock(macros={})) - self.assertEqual(query_header.add(self.query), self.query) + assert query_header.add(query) == query From 4c587544b65bb3f4bafddc37562aebf34e9c4a7a Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 18 Apr 2024 11:54:22 -0400 Subject: [PATCH 15/17] clear DBT_DEBUG after test_env_vars_secrets (#9978) --- tests/functional/context_methods/test_env_vars.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/context_methods/test_env_vars.py b/tests/functional/context_methods/test_env_vars.py index 506ed40d31c..33feb3b5de1 100644 --- a/tests/functional/context_methods/test_env_vars.py +++ b/tests/functional/context_methods/test_env_vars.py @@ -191,3 +191,4 @@ def test_env_vars_secrets(self, project): assert not ("secret_variable" in log_output) assert "regular_variable" in log_output + del os.environ["DBT_DEBUG"] From 514647b29f40a0049850b88d03ffaf1da7502f74 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 18 Apr 2024 15:59:50 +0000 Subject: [PATCH 16/17] Bumping version to 1.8.0b3 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.8.0-b3.md | 48 +++++++++++++++++ .../Dependencies-20240117-100818.yaml | 0 .../Dependencies-20240227-142138.yaml | 0 .../Dependencies-20240331-103917.yaml | 0 .../Dependencies-20240410-183321.yaml | 0 .../Features-20240307-153622.yaml | 0 .../Features-20240323-201230.yaml | 0 .../Features-20240404-170728.yaml | 0 .../Features-20240405-175733.yaml | 0 .../Features-20240408-094132.yaml | 0 .../Fixes-20240108-232035.yaml | 0 .../Fixes-20240206-152435.yaml | 0 .../Fixes-20240323-124558.yaml | 0 .../Fixes-20240402-135556.yaml | 0 .../Fixes-20240408-130646.yaml | 0 .../Fixes-20240409-233347.yaml | 0 .../Fixes-20240412-095718.yaml | 0 .../Security-20240417-141316.yaml | 0 .../Under the Hood-20240412-132000.yaml | 0 .../Under the Hood-20240412-134502.yaml | 0 .../Under the Hood-20240416-150030.yaml | 0 CHANGELOG.md | 51 ++++++++++++++++++- core/dbt/version.py | 2 +- core/setup.py | 2 +- 25 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 .changes/1.8.0-b3.md rename .changes/{unreleased => 1.8.0}/Dependencies-20240117-100818.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240227-142138.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240331-103917.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20240410-183321.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240307-153622.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240323-201230.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240404-170728.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240405-175733.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240408-094132.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240108-232035.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240206-152435.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240323-124558.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240402-135556.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240408-130646.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240409-233347.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240412-095718.yaml (100%) rename .changes/{unreleased => 1.8.0}/Security-20240417-141316.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240412-132000.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240412-134502.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240416-150030.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6e036340dd6..b9422be0685 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0b2 +current_version = 1.8.0b3 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b3.md b/.changes/1.8.0-b3.md new file mode 100644 index 00000000000..0b9ce6aaaca --- /dev/null +++ b/.changes/1.8.0-b3.md @@ -0,0 +1,48 @@ +## dbt-core 1.8.0-b3 - April 18, 2024 + +### Features + +- Support scrubbing secret vars ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247)) +- Add wildcard support to the group selector method ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811)) +- source freshness precomputes metadata-based freshness in batch, if possible ([#8705](https://github.com/dbt-labs/dbt-core/issues/8705)) +- Better error message when trying to select a disabled model ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747)) +- Support SQL in unit testing fixtures ([#9405](https://github.com/dbt-labs/dbt-core/issues/9405)) + +### Fixes + +- fix configuration of turning test warnings into failures with WARN_ERROR_OPTIONS ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761)) +- Fix conflict with newer versions of Snowplow tracker ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- Only create the packages-install-path / dbt_packages folder during dbt deps ([#6985](https://github.com/dbt-labs/dbt-core/issues/6985), [#9584](https://github.com/dbt-labs/dbt-core/issues/9584)) +- Exclude password-like fields for considering reparse ([#9795](https://github.com/dbt-labs/dbt-core/issues/9795)) +- Fixed query comments test ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860)) +- Begin warning people about spaces in model names ([#9397](https://github.com/dbt-labs/dbt-core/issues/9397)) +- Disambiguiate FreshnessConfigProblem error message ([#9891](https://github.com/dbt-labs/dbt-core/issues/9891)) + +### Under the Hood + +- Remove non dbt.artifacts dbt.* imports from dbt/artifacts ([#9926](https://github.com/dbt-labs/dbt-core/issues/9926)) +- Migrate to using `error_tag` provided by `dbt-common` ([#9914](https://github.com/dbt-labs/dbt-core/issues/9914)) +- Add a test for semantic manifest and move test fixtures needed for it ([#9665](https://github.com/dbt-labs/dbt-core/issues/9665)) + +### Dependencies + +- Relax pathspec upper bound version restriction ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373)) +- Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker ([#9687](https://github.com/dbt-labs/dbt-core/issues/9687)) +- Remove duplicate dependency of protobuf in dev-requirements ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830)) +- Bump black from 23.3.0 to >=24.3.0,<25.0 ([#8074](https://github.com/dbt-labs/dbt-core/issues/8074)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951)) + +### Contributors +- [@SamuelBFavarin](https://github.com/SamuelBFavarin) ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747)) +- [@akurdyukov](https://github.com/akurdyukov) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- [@damian3031](https://github.com/damian3031) ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860)) +- [@edgarrmondragon](https://github.com/edgarrmondragon) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- [@emmoop](https://github.com/emmoop) ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951)) +- [@heysweet](https://github.com/heysweet) ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811)) +- [@jx2lee](https://github.com/jx2lee) ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761)) +- [@nielspardon](https://github.com/nielspardon) ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247)) +- [@niteshy](https://github.com/niteshy) ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830)) +- [@rzjfr](https://github.com/rzjfr) ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373)) diff --git a/.changes/unreleased/Dependencies-20240117-100818.yaml b/.changes/1.8.0/Dependencies-20240117-100818.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240117-100818.yaml rename to .changes/1.8.0/Dependencies-20240117-100818.yaml diff --git a/.changes/unreleased/Dependencies-20240227-142138.yaml b/.changes/1.8.0/Dependencies-20240227-142138.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240227-142138.yaml rename to .changes/1.8.0/Dependencies-20240227-142138.yaml diff --git a/.changes/unreleased/Dependencies-20240331-103917.yaml b/.changes/1.8.0/Dependencies-20240331-103917.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240331-103917.yaml rename to .changes/1.8.0/Dependencies-20240331-103917.yaml diff --git a/.changes/unreleased/Dependencies-20240410-183321.yaml b/.changes/1.8.0/Dependencies-20240410-183321.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240410-183321.yaml rename to .changes/1.8.0/Dependencies-20240410-183321.yaml diff --git a/.changes/unreleased/Features-20240307-153622.yaml b/.changes/1.8.0/Features-20240307-153622.yaml similarity index 100% rename from .changes/unreleased/Features-20240307-153622.yaml rename to .changes/1.8.0/Features-20240307-153622.yaml diff --git a/.changes/unreleased/Features-20240323-201230.yaml b/.changes/1.8.0/Features-20240323-201230.yaml similarity index 100% rename from .changes/unreleased/Features-20240323-201230.yaml rename to .changes/1.8.0/Features-20240323-201230.yaml diff --git a/.changes/unreleased/Features-20240404-170728.yaml b/.changes/1.8.0/Features-20240404-170728.yaml similarity index 100% rename from .changes/unreleased/Features-20240404-170728.yaml rename to .changes/1.8.0/Features-20240404-170728.yaml diff --git a/.changes/unreleased/Features-20240405-175733.yaml b/.changes/1.8.0/Features-20240405-175733.yaml similarity index 100% rename from .changes/unreleased/Features-20240405-175733.yaml rename to .changes/1.8.0/Features-20240405-175733.yaml diff --git a/.changes/unreleased/Features-20240408-094132.yaml b/.changes/1.8.0/Features-20240408-094132.yaml similarity index 100% rename from .changes/unreleased/Features-20240408-094132.yaml rename to .changes/1.8.0/Features-20240408-094132.yaml diff --git a/.changes/unreleased/Fixes-20240108-232035.yaml b/.changes/1.8.0/Fixes-20240108-232035.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240108-232035.yaml rename to .changes/1.8.0/Fixes-20240108-232035.yaml diff --git a/.changes/unreleased/Fixes-20240206-152435.yaml b/.changes/1.8.0/Fixes-20240206-152435.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240206-152435.yaml rename to .changes/1.8.0/Fixes-20240206-152435.yaml diff --git a/.changes/unreleased/Fixes-20240323-124558.yaml b/.changes/1.8.0/Fixes-20240323-124558.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240323-124558.yaml rename to .changes/1.8.0/Fixes-20240323-124558.yaml diff --git a/.changes/unreleased/Fixes-20240402-135556.yaml b/.changes/1.8.0/Fixes-20240402-135556.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240402-135556.yaml rename to .changes/1.8.0/Fixes-20240402-135556.yaml diff --git a/.changes/unreleased/Fixes-20240408-130646.yaml b/.changes/1.8.0/Fixes-20240408-130646.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240408-130646.yaml rename to .changes/1.8.0/Fixes-20240408-130646.yaml diff --git a/.changes/unreleased/Fixes-20240409-233347.yaml b/.changes/1.8.0/Fixes-20240409-233347.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240409-233347.yaml rename to .changes/1.8.0/Fixes-20240409-233347.yaml diff --git a/.changes/unreleased/Fixes-20240412-095718.yaml b/.changes/1.8.0/Fixes-20240412-095718.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240412-095718.yaml rename to .changes/1.8.0/Fixes-20240412-095718.yaml diff --git a/.changes/unreleased/Security-20240417-141316.yaml b/.changes/1.8.0/Security-20240417-141316.yaml similarity index 100% rename from .changes/unreleased/Security-20240417-141316.yaml rename to .changes/1.8.0/Security-20240417-141316.yaml diff --git a/.changes/unreleased/Under the Hood-20240412-132000.yaml b/.changes/1.8.0/Under the Hood-20240412-132000.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240412-132000.yaml rename to .changes/1.8.0/Under the Hood-20240412-132000.yaml diff --git a/.changes/unreleased/Under the Hood-20240412-134502.yaml b/.changes/1.8.0/Under the Hood-20240412-134502.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240412-134502.yaml rename to .changes/1.8.0/Under the Hood-20240412-134502.yaml diff --git a/.changes/unreleased/Under the Hood-20240416-150030.yaml b/.changes/1.8.0/Under the Hood-20240416-150030.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240416-150030.yaml rename to .changes/1.8.0/Under the Hood-20240416-150030.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 29682dc6640..8c4e9dcf37f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,56 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-core 1.8.0-b3 - April 18, 2024 + +### Features + +- Support scrubbing secret vars ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247)) +- Add wildcard support to the group selector method ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811)) +- source freshness precomputes metadata-based freshness in batch, if possible ([#8705](https://github.com/dbt-labs/dbt-core/issues/8705)) +- Better error message when trying to select a disabled model ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747)) +- Support SQL in unit testing fixtures ([#9405](https://github.com/dbt-labs/dbt-core/issues/9405)) + +### Fixes + +- fix configuration of turning test warnings into failures with WARN_ERROR_OPTIONS ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761)) +- Fix conflict with newer versions of Snowplow tracker ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- Only create the packages-install-path / dbt_packages folder during dbt deps ([#6985](https://github.com/dbt-labs/dbt-core/issues/6985), [#9584](https://github.com/dbt-labs/dbt-core/issues/9584)) +- Exclude password-like fields for considering reparse ([#9795](https://github.com/dbt-labs/dbt-core/issues/9795)) +- Fixed query comments test ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860)) +- Begin warning people about spaces in model names ([#9397](https://github.com/dbt-labs/dbt-core/issues/9397)) +- Disambiguiate FreshnessConfigProblem error message ([#9891](https://github.com/dbt-labs/dbt-core/issues/9891)) + +### Under the Hood + +- Remove non dbt.artifacts dbt.* imports from dbt/artifacts ([#9926](https://github.com/dbt-labs/dbt-core/issues/9926)) +- Migrate to using `error_tag` provided by `dbt-common` ([#9914](https://github.com/dbt-labs/dbt-core/issues/9914)) +- Add a test for semantic manifest and move test fixtures needed for it ([#9665](https://github.com/dbt-labs/dbt-core/issues/9665)) + +### Dependencies + +- Relax pathspec upper bound version restriction ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373)) +- Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker ([#9687](https://github.com/dbt-labs/dbt-core/issues/9687)) +- Remove duplicate dependency of protobuf in dev-requirements ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830)) +- Bump black from 23.3.0 to >=24.3.0,<25.0 ([#8074](https://github.com/dbt-labs/dbt-core/issues/8074)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951)) + +### Contributors +- [@SamuelBFavarin](https://github.com/SamuelBFavarin) ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747)) +- [@akurdyukov](https://github.com/akurdyukov) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- [@damian3031](https://github.com/damian3031) ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860)) +- [@edgarrmondragon](https://github.com/edgarrmondragon) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719)) +- [@emmoop](https://github.com/emmoop) ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951)) +- [@heysweet](https://github.com/heysweet) ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811)) +- [@jx2lee](https://github.com/jx2lee) ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761)) +- [@nielspardon](https://github.com/nielspardon) ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247)) +- [@niteshy](https://github.com/niteshy) ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830)) +- [@rzjfr](https://github.com/rzjfr) ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373)) + + ## dbt-core 1.8.0-b2 - April 03, 2024 ### Features @@ -59,7 +109,6 @@ - [@jx2lee](https://github.com/jx2lee) ([#9319](https://github.com/dbt-labs/dbt-core/issues/9319)) - [@slothkong](https://github.com/slothkong) ([#9570](https://github.com/dbt-labs/dbt-core/issues/9570)) - ## dbt-core 1.8.0-b1 - February 28, 2024 ### Breaking Changes diff --git a/core/dbt/version.py b/core/dbt/version.py index 13d31eaa632..5d515185ae6 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -229,5 +229,5 @@ def _get_adapter_plugin_names() -> Iterator[str]: yield plugin_name -__version__ = "1.8.0b2" +__version__ = "1.8.0b3" installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 83b150b1f74..1f103c19534 100644 --- a/core/setup.py +++ b/core/setup.py @@ -25,7 +25,7 @@ package_name = "dbt-core" -package_version = "1.8.0b2" +package_version = "1.8.0b3" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" From 4811ada35a8484b526173cb84bbdd5036110aa8e Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 19 Apr 2024 11:10:37 +0100 Subject: [PATCH 17/17] Fix #9534: Add NodeRelation to SavedQuery Export (#9794) --- .../unreleased/Fixes-20240410-181741.yaml | 6 ++++ .../dbt/artifacts/resources/v1/saved_query.py | 1 + core/dbt/parser/base.py | 6 ++-- core/dbt/parser/schema_yaml_readers.py | 16 ++++++++++ tests/functional/saved_queries/fixtures.py | 23 ++++++++++++++ .../functional/saved_queries/test_configs.py | 30 +++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240410-181741.yaml diff --git a/.changes/unreleased/Fixes-20240410-181741.yaml b/.changes/unreleased/Fixes-20240410-181741.yaml new file mode 100644 index 00000000000..66ec5e7d373 --- /dev/null +++ b/.changes/unreleased/Fixes-20240410-181741.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Add NodeRelation to SavedQuery Export +time: 2024-04-10T18:17:41.42533+01:00 +custom: + Author: aranke + Issue: "9534" diff --git a/core/dbt/artifacts/resources/v1/saved_query.py b/core/dbt/artifacts/resources/v1/saved_query.py index cc24d8fddf4..5f0575d26a7 100644 --- a/core/dbt/artifacts/resources/v1/saved_query.py +++ b/core/dbt/artifacts/resources/v1/saved_query.py @@ -21,6 +21,7 @@ class ExportConfig(dbtClassMixin): export_as: ExportDestinationType schema_name: Optional[str] = None alias: Optional[str] = None + database: Optional[str] = None @dataclass diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 61e34237e5c..e345a74183c 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -109,7 +109,7 @@ def __init__(self, config: RuntimeConfig, manifest: Manifest, component: str) -> self.component = component def __call__(self, parsed_node: Any, override: Optional[str]) -> None: - if parsed_node.package_name in self.package_updaters: + if getattr(parsed_node, "package_name", None) in self.package_updaters: new_value = self.package_updaters[parsed_node.package_name](override, parsed_node) else: new_value = self.default_updater(override, parsed_node) @@ -293,7 +293,7 @@ def update_parsed_node_relation_names( self._update_node_alias(parsed_node, config_dict.get("alias")) # Snapshot nodes use special "target_database" and "target_schema" fields for some reason - if parsed_node.resource_type == NodeType.Snapshot: + if getattr(parsed_node, "resource_type", None) == NodeType.Snapshot: if "target_database" in config_dict and config_dict["target_database"]: parsed_node.database = config_dict["target_database"] if "target_schema" in config_dict and config_dict["target_schema"]: @@ -452,7 +452,7 @@ def _update_node_relation_name(self, node: ManifestNode): # and TestNodes that store_failures. # TestNodes do not get a relation_name without store failures # because no schema is created. - if node.is_relational and not node.is_ephemeral_model: + if getattr(node, "is_relational", None) and not getattr(node, "is_ephemeral_model", None): adapter = get_adapter(self.root_project) relation_cls = adapter.Relation node.relation_name = str(relation_cls.create_from(self.root_project, node)) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 8b1a780a0c6..b7c047d01dd 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -764,6 +764,22 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: group=config.group, ) + for export in parsed.exports: + self.schema_parser.update_parsed_node_relation_names(export, export.config.to_dict()) # type: ignore + + if not export.config.schema_name: + export.config.schema_name = getattr(export, "schema", None) + delattr(export, "schema") + + export.config.database = getattr(export, "database", None) or export.config.database + delattr(export, "database") + + if not export.config.alias: + export.config.alias = getattr(export, "alias", None) + delattr(export, "alias") + + delattr(export, "relation_name") + # Only add thes saved query if it's enabled, otherwise we track it with other diabled nodes if parsed.config.enabled: self.manifest.add_saved_query(self.yaml.file, parsed) diff --git a/tests/functional/saved_queries/fixtures.py b/tests/functional/saved_queries/fixtures.py index b2025ba208a..e938760a12e 100644 --- a/tests/functional/saved_queries/fixtures.py +++ b/tests/functional/saved_queries/fixtures.py @@ -26,6 +26,29 @@ schema: my_export_schema_name """ +saved_queries_with_defaults_yml = """ +version: 2 + +saved_queries: + - name: test_saved_query + description: "{{ doc('saved_query_description') }}" + label: Test Saved Query + query_params: + metrics: + - simple_metric + group_by: + - "Dimension('user__ds')" + where: + - "{{ Dimension('user__ds', 'DAY') }} <= now()" + - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" + - "{{ Metric('txn_revenue', ['id']) }} > 1" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table +""" + saved_queries_with_diff_filters_yml = """ version: 2 diff --git a/tests/functional/saved_queries/test_configs.py b/tests/functional/saved_queries/test_configs.py index 4c55c54a9eb..ef63888441a 100644 --- a/tests/functional/saved_queries/test_configs.py +++ b/tests/functional/saved_queries/test_configs.py @@ -12,6 +12,7 @@ saved_query_with_extra_config_attributes_yml, saved_query_with_export_configs_defined_at_saved_query_level_yml, saved_query_without_export_configs_defined_yml, + saved_queries_with_defaults_yml, ) from tests.functional.semantic_models.fixtures import ( fct_revenue_sql, @@ -121,6 +122,33 @@ def test_extra_config_properties_dont_break_parsing(self, project): assert saved_query.exports[0].config.__dict__.get("my_random_config") is None +class TestExportConfigsWithDefaultProperties(BaseConfigProject): + @pytest.fixture(scope="class") + def models(self): + return { + "saved_queries.yml": saved_queries_with_defaults_yml, + "schema.yml": schema_yml, + "fct_revenue.sql": fct_revenue_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "docs.md": saved_query_description, + } + + def test_default_properties(self, project): + runner = dbtTestRunner() + + # parse with default fixture project config + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + assert len(result.result.saved_queries) == 1 + saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] + assert len(saved_query.exports) == 1 + export = saved_query.exports[0] + assert export.config.alias == "my_export_alias" + assert export.config.schema_name == project.test_schema + assert export.config.database == project.database + + class TestInheritingExportConfigFromSavedQueryConfig(BaseConfigProject): @pytest.fixture(scope="class") def models(self): @@ -152,6 +180,7 @@ def test_export_config_inherits_from_saved_query(self, project): assert export1.config.export_as != saved_query.config.export_as assert export1.config.schema_name == "my_custom_export_schema" assert export1.config.schema_name != saved_query.config.schema + assert export1.config.database == project.database # assert Export `my_export` has its configs defined from the saved_query because they should take priority export2 = next( @@ -162,6 +191,7 @@ def test_export_config_inherits_from_saved_query(self, project): assert export2.config.export_as == saved_query.config.export_as assert export2.config.schema_name == "my_default_export_schema" assert export2.config.schema_name == saved_query.config.schema + assert export2.config.database == project.database class TestInheritingExportConfigsFromProject(BaseConfigProject):