Merge branch 'main' into cl/update_to_pytest

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.8.0b2
+current_version = 1.8.0b3
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number
@@ -35,5 +35,3 @@ first_value = 1
 [bumpversion:file:core/setup.py]
 
 [bumpversion:file:core/dbt/version.py]
-
-[bumpversion:file:docker/Dockerfile]

.changes/1.8.0-b3.md

@@ -0,0 +1,48 @@
+## dbt-core 1.8.0-b3 - April 18, 2024
+
+### Features
+
+- Support scrubbing secret vars ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247))
+- Add wildcard support to the group selector method ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811))
+- source freshness precomputes metadata-based freshness in batch, if possible  ([#8705](https://github.com/dbt-labs/dbt-core/issues/8705))
+- Better error message when trying to select a disabled model ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747))
+- Support SQL in unit testing fixtures ([#9405](https://github.com/dbt-labs/dbt-core/issues/9405))
+
+### Fixes
+
+- fix configuration of turning test warnings into failures with WARN_ERROR_OPTIONS ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761))
+- Fix conflict with newer versions of Snowplow tracker ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- Only create the packages-install-path / dbt_packages folder during dbt deps ([#6985](https://github.com/dbt-labs/dbt-core/issues/6985), [#9584](https://github.com/dbt-labs/dbt-core/issues/9584))
+- Exclude password-like fields for considering reparse ([#9795](https://github.com/dbt-labs/dbt-core/issues/9795))
+- Fixed query comments test ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860))
+- Begin warning people about spaces in model names ([#9397](https://github.com/dbt-labs/dbt-core/issues/9397))
+- Disambiguiate FreshnessConfigProblem error message ([#9891](https://github.com/dbt-labs/dbt-core/issues/9891))
+
+### Under the Hood
+
+- Remove non dbt.artifacts dbt.* imports from dbt/artifacts ([#9926](https://github.com/dbt-labs/dbt-core/issues/9926))
+- Migrate to using `error_tag` provided by `dbt-common` ([#9914](https://github.com/dbt-labs/dbt-core/issues/9914))
+- Add a test for semantic manifest and move test fixtures needed for it ([#9665](https://github.com/dbt-labs/dbt-core/issues/9665))
+
+### Dependencies
+
+- Relax pathspec upper bound version restriction ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373))
+- Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker ([#9687](https://github.com/dbt-labs/dbt-core/issues/9687))
+- Remove duplicate dependency of protobuf in dev-requirements ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830))
+- Bump black from 23.3.0 to >=24.3.0,<25.0 ([#8074](https://github.com/dbt-labs/dbt-core/issues/8074))
+
+### Security
+
+- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951))
+
+### Contributors
+- [@SamuelBFavarin](https://github.com/SamuelBFavarin) ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747))
+- [@akurdyukov](https://github.com/akurdyukov) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- [@damian3031](https://github.com/damian3031) ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860))
+- [@edgarrmondragon](https://github.com/edgarrmondragon) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- [@emmoop](https://github.com/emmoop) ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951))
+- [@heysweet](https://github.com/heysweet) ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811))
+- [@jx2lee](https://github.com/jx2lee) ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761))
+- [@nielspardon](https://github.com/nielspardon) ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247))
+- [@niteshy](https://github.com/niteshy) ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830))
+- [@rzjfr](https://github.com/rzjfr) ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373))

.changes/1.8.0/Dependencies-20240227-142138.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker
+time: 2024-02-27T14:21:38.394757-05:00
+custom:
+  Author: michelleark
+  Issue: "9687"

.changes/1.8.0/Dependencies-20240410-183321.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump black from 23.3.0 to >=24.3.0,<25.0"
+time: 2024-04-10T18:33:21.00000Z
+custom:
+  Author: dependabot[bot]
+  Issue: 8074

.changes/1.8.0/Features-20240307-153622.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support scrubbing secret vars
+time: 2024-03-07T15:36:22.754627+01:00
+custom:
+  Author: nielspardon
+  Issue: "7247"

.changes/1.8.0/Features-20240408-094132.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support SQL in unit testing fixtures
+time: 2024-04-08T09:41:32.15936-04:00
+custom:
+  Author: gshank
+  Issue: "9405"

.changes/1.8.0/Fixes-20240408-130646.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fixed query comments test
+time: 2024-04-08T13:06:46.648144+02:00
+custom:
+  Author: damian3031
+  Issue: "9860"

.changes/1.8.0/Security-20240417-141316.yaml

@@ -0,0 +1,6 @@
+kind: Security
+body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg
+time: 2024-04-17T14:13:16.896353-05:00
+custom:
+  Author: emmoop
+  Issue: "9951"

.changes/1.8.0/Under the Hood-20240416-150030.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add a test for semantic manifest and move test fixtures needed for it
+time: 2024-04-16T15:00:30.614286-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "9665"

.changes/unreleased/Fixes-20240410-181741.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Add NodeRelation to SavedQuery Export
+time: 2024-04-10T18:17:41.42533+01:00
+custom:
+  Author: aranke
+  Issue: "9534"

.github/workflows/nightly-release.yml

@@ -20,6 +20,7 @@ on:
 
 permissions:
   contents: write # this is the permission that allows creating a new release
+  packages: write # this is the permission that allows Docker release
 
 defaults:
   run:

.github/workflows/release.yml

@@ -39,6 +39,11 @@ on:
         type: boolean
         default: false
         required: false
+      only_docker:
+        description: "Only release Docker image, skip GitHub & PyPI"
+        type: boolean
+        default: false
+        required: false
   workflow_call:
     inputs:
       target_branch:
@@ -81,6 +86,7 @@ jobs:
           echo The release version number:         ${{ inputs.version_number }}
           echo Test run:                           ${{ inputs.test_run }}
           echo Nightly release:                    ${{ inputs.nightly_release }}
+          echo Only Docker:                        ${{ inputs.only_docker }}
 
       - name: "Checkout target branch"
         uses: actions/checkout@v4
@@ -99,6 +105,7 @@ jobs:
   bump-version-generate-changelog:
     name: Bump package version, Generate changelog
     needs: [job-setup]
+    if: ${{ !inputs.only_docker }}
 
     uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
 
@@ -114,7 +121,7 @@ jobs:
 
   log-outputs-bump-version-generate-changelog:
     name: "[Log output] Bump package version, Generate changelog"
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
 
     needs: [bump-version-generate-changelog]
 
@@ -128,7 +135,7 @@ jobs:
 
   build-test-package:
     name: Build, Test, Package
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
     needs: [job-setup, bump-version-generate-changelog]
 
     uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
@@ -149,7 +156,7 @@ jobs:
 
   github-release:
     name: GitHub Release
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
 
     needs: [bump-version-generate-changelog, build-test-package]
 
@@ -180,6 +187,7 @@ jobs:
     # dbt-postgres exists within dbt-core for versions 1.7 and earlier but is a separate package for 1.8 and later.
     # determine if we need to release dbt-core or both dbt-core and dbt-postgres
     name: Determine Docker Package
+    if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-latest
     needs: [pypi-release]
     outputs:
@@ -204,6 +212,10 @@ jobs:
   docker-release:
     name: "Docker Release for ${{ matrix.package }}"
     needs: [determine-docker-package]
+    # We cannot release to docker on a test run because it uses the tag in GitHub as
+    # what we need to release but draft releases don't actually tag the commit so it
+    # finds nothing to release
+    if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
     strategy:
       matrix: ${{fromJson(needs.determine-docker-package.outputs.matrix)}}
 

CHANGELOG.md

@@ -5,6 +5,56 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-core 1.8.0-b3 - April 18, 2024
+
+### Features
+
+- Support scrubbing secret vars ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247))
+- Add wildcard support to the group selector method ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811))
+- source freshness precomputes metadata-based freshness in batch, if possible  ([#8705](https://github.com/dbt-labs/dbt-core/issues/8705))
+- Better error message when trying to select a disabled model ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747))
+- Support SQL in unit testing fixtures ([#9405](https://github.com/dbt-labs/dbt-core/issues/9405))
+
+### Fixes
+
+- fix configuration of turning test warnings into failures with WARN_ERROR_OPTIONS ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761))
+- Fix conflict with newer versions of Snowplow tracker ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- Only create the packages-install-path / dbt_packages folder during dbt deps ([#6985](https://github.com/dbt-labs/dbt-core/issues/6985), [#9584](https://github.com/dbt-labs/dbt-core/issues/9584))
+- Exclude password-like fields for considering reparse ([#9795](https://github.com/dbt-labs/dbt-core/issues/9795))
+- Fixed query comments test ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860))
+- Begin warning people about spaces in model names ([#9397](https://github.com/dbt-labs/dbt-core/issues/9397))
+- Disambiguiate FreshnessConfigProblem error message ([#9891](https://github.com/dbt-labs/dbt-core/issues/9891))
+
+### Under the Hood
+
+- Remove non dbt.artifacts dbt.* imports from dbt/artifacts ([#9926](https://github.com/dbt-labs/dbt-core/issues/9926))
+- Migrate to using `error_tag` provided by `dbt-common` ([#9914](https://github.com/dbt-labs/dbt-core/issues/9914))
+- Add a test for semantic manifest and move test fixtures needed for it ([#9665](https://github.com/dbt-labs/dbt-core/issues/9665))
+
+### Dependencies
+
+- Relax pathspec upper bound version restriction ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373))
+- Bump python from 3.10.7-slim-nullseye to 3.11.2-slim-bullseye in /docker ([#9687](https://github.com/dbt-labs/dbt-core/issues/9687))
+- Remove duplicate dependency of protobuf in dev-requirements ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830))
+- Bump black from 23.3.0 to >=24.3.0,<25.0 ([#8074](https://github.com/dbt-labs/dbt-core/issues/8074))
+
+### Security
+
+- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951))
+
+### Contributors
+- [@SamuelBFavarin](https://github.com/SamuelBFavarin) ([#9747](https://github.com/dbt-labs/dbt-core/issues/9747))
+- [@akurdyukov](https://github.com/akurdyukov) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- [@damian3031](https://github.com/damian3031) ([#9860](https://github.com/dbt-labs/dbt-core/issues/9860))
+- [@edgarrmondragon](https://github.com/edgarrmondragon) ([#8719](https://github.com/dbt-labs/dbt-core/issues/8719))
+- [@emmoop](https://github.com/emmoop) ([#9951](https://github.com/dbt-labs/dbt-core/issues/9951))
+- [@heysweet](https://github.com/heysweet) ([#9811](https://github.com/dbt-labs/dbt-core/issues/9811))
+- [@jx2lee](https://github.com/jx2lee) ([#7761](https://github.com/dbt-labs/dbt-core/issues/7761))
+- [@nielspardon](https://github.com/nielspardon) ([#7247](https://github.com/dbt-labs/dbt-core/issues/7247))
+- [@niteshy](https://github.com/niteshy) ([#9830](https://github.com/dbt-labs/dbt-core/issues/9830))
+- [@rzjfr](https://github.com/rzjfr) ([#9373](https://github.com/dbt-labs/dbt-core/issues/9373))
+
+
 ## dbt-core 1.8.0-b2 - April 03, 2024
 
 ### Features
@@ -59,7 +109,6 @@
 - [@jx2lee](https://github.com/jx2lee) ([#9319](https://github.com/dbt-labs/dbt-core/issues/9319))
 - [@slothkong](https://github.com/slothkong) ([#9570](https://github.com/dbt-labs/dbt-core/issues/9570))
 
-
 ## dbt-core 1.8.0-b1 - February 28, 2024
 
 ### Breaking Changes

core/dbt/artifacts/resources/v1/saved_query.py

@@ -21,6 +21,7 @@ class ExportConfig(dbtClassMixin):
     export_as: ExportDestinationType
     schema_name: Optional[str] = None
     alias: Optional[str] = None
+    database: Optional[str] = None
 
 
 @dataclass

core/dbt/artifacts/resources/v1/semantic_model.py

@@ -42,7 +42,7 @@ class NodeRelation(dbtClassMixin):
     alias: str
     schema_name: str  # TODO: Could this be called simply "schema" so we could reuse StateRelation?
     database: Optional[str] = None
-    relation_name: Optional[str] = None
+    relation_name: Optional[str] = ""
 
 
 # ====================================

core/dbt/artifacts/resources/v1/unit_test_definition.py

@@ -30,6 +30,7 @@ class UnitTestConfig(BaseConfig):
 class UnitTestFormat(StrEnum):
     CSV = "csv"
     Dict = "dict"
+    SQL = "sql"
 
 
 @dataclass

core/dbt/artifacts/schemas/run/v5/run.py

@@ -1,9 +1,11 @@
 import threading
 from typing import Any, Optional, Iterable, Tuple, Sequence, Dict, TYPE_CHECKING
+import copy
 from dataclasses import dataclass, field
 from datetime import datetime
 
 
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.artifacts.resources import CompiledResource
 from dbt.artifacts.schemas.base import (
     BaseArtifactMetadata,
@@ -19,6 +21,7 @@
     ExecutionResult,
 )
 from dbt_common.clients.system import write_json
+from dbt.exceptions import scrub_secrets
 
 
 if TYPE_CHECKING:
@@ -123,7 +126,26 @@ def from_execution_results(
             dbt_schema_version=str(cls.dbt_schema_version),
             generated_at=generated_at,
         )
-        return cls(metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=args)
+
+        secret_vars = [
+            v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
+        ]
+
+        scrubbed_args = copy.deepcopy(args)
+
+        # scrub secrets in invocation command
+        scrubbed_args["invocation_command"] = scrub_secrets(
+            scrubbed_args["invocation_command"], secret_vars
+        )
+
+        # scrub secrets in vars dict
+        scrubbed_args["vars"] = {
+            k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items()
+        }
+
+        return cls(
+            metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args
+        )
 
     @classmethod
     def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:

core/dbt/cli/requires.py

@@ -15,7 +15,7 @@
 from dbt.cli.flags import Flags
 from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile, UnsetProfile
-from dbt.context.manifest import generate_query_header_context
+from dbt.context.query_header import generate_query_header_context
 
 from dbt_common.events.base_types import EventLevel
 from dbt_common.events.functions import (

core/dbt/context/manifest.py

@@ -71,13 +71,3 @@ def to_dict(self):
     @contextproperty()
     def context_macro_stack(self):
         return self.macro_stack
-
-
-class QueryHeaderContext(ManifestContext):
-    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None:
-        super().__init__(config, manifest, config.project_name)
-
-
-def generate_query_header_context(config: AdapterRequiredConfig, manifest: Manifest):
-    ctx = QueryHeaderContext(config, manifest)
-    return ctx.to_dict()

core/dbt/context/query_header.py

@@ -0,0 +1,13 @@
+from dbt.adapters.contracts.connection import AdapterRequiredConfig
+from dbt.context.manifest import ManifestContext
+from dbt.contracts.graph.manifest import Manifest
+
+
+class QueryHeaderContext(ManifestContext):
+    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None:
+        super().__init__(config, manifest, config.project_name)
+
+
+def generate_query_header_context(config: AdapterRequiredConfig, manifest: Manifest):
+    ctx = QueryHeaderContext(config, manifest)
+    return ctx.to_dict()

core/dbt/contracts/graph/model_config.py

@@ -36,6 +36,7 @@ def insensitive_patterns(*patterns: str):
 @dataclass
 class UnitTestNodeConfig(NodeConfig):
     expected_rows: List[Dict[str, Any]] = field(default_factory=list)
+    expected_sql: Optional[str] = None
 
 
 @dataclass

core/dbt/contracts/graph/nodes.py

@@ -991,7 +991,7 @@ def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool:
 @dataclass
 class UnitTestFileFixture(BaseNode):
     resource_type: Literal[NodeType.Fixture]
-    rows: Optional[List[Dict[str, Any]]] = None
+    rows: Optional[Union[List[Dict[str, Any]], str]] = None
 
 
 # ====================================