Merge remote-tracking branch 'origin/main' into jerco/refactor-refact…

…or-deferral

.changes/unreleased/Breaking Changes-20240430-165247.yaml

@@ -0,0 +1,7 @@
+kind: Breaking Changes
+body: Update the default behaviour of require_explicit_package_overrides_for_builtin_materializations
+  to True.
+time: 2024-04-30T16:52:47.610025-04:00
+custom:
+  Author: jtcohen6
+  Issue: "10062"

.changes/unreleased/Docs-20240311-140344.yaml

@@ -0,0 +1,6 @@
+kind: Docs
+body: Enable display of unit tests
+time: 2024-03-11T14:03:44.490834-04:00
+custom:
+  Author: gshank
+  Issue: "501"

.changes/unreleased/Docs-20240501-021050.yaml

@@ -0,0 +1,6 @@
+kind: Docs
+body: Unit tests not rendering
+time: 2024-05-01T02:10:50.987412+02:00
+custom:
+  Author: aranke
+  Issue: "506"

.changes/unreleased/Features-20231212-163409.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Package selector syntax for the current package
+time: 2023-12-12T16:34:09.328891Z
+custom:
+  Author: barton996
+  Issue: "6891"

.changes/unreleased/Features-20240424-180639.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: add --empty flag to dbt build command
+time: 2024-04-24T18:06:39.438457-04:00
+custom:
+  Author: michelleark
+  Issue: "10026"

.changes/unreleased/Features-20240426-233126.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Ability to `silence` warnings via `warn_error_options`
+time: 2024-04-26T23:31:26.601057-05:00
+custom:
+  Author: QMalcolm
+  Issue: "9644"

.changes/unreleased/Features-20240426-233208.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Allow aliases `error` for `include` and `warn` for `exclude` in `warn_error_options`
+time: 2024-04-26T23:32:08.771114-05:00
+custom:
+  Author: QMalcolm
+  Issue: "9644"

.changes/unreleased/Fixes-20240422-145811.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Use consistent secret scrubbing with the log function.
+time: 2024-04-22T14:58:11.990326-05:00
+custom:
+  Author: emmyoop
+  Issue: "9987"

.changes/unreleased/Fixes-20240422-152244.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Validate against empty strings in package definitions
+time: 2024-04-22T15:22:44.575999-05:00
+custom:
+  Author: emmyoop
+  Issue: "9985"

.changes/unreleased/Fixes-20240422-173532.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix default value for indirect selection in selector cannot overwritten by CLI
+  flag and env var
+time: 2024-04-22T17:35:32.465183-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: 9976 7673

.changes/unreleased/Under the Hood-20240423-143055.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Use the SECRET_ENV_PREFIX from dbt_common instead of duplicating it in dbt-core
+time: 2024-04-23T14:30:55.739378-05:00
+custom:
+  Author: emmyoop
+  Issue: "10018"

.changes/unreleased/Under the Hood-20240425-170138.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Enable use of record mode via environment variable
+time: 2024-04-25T17:01:38.093524-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "10045"

.changes/unreleased/Under the Hood-20240429-142342.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Consistent naming + deprecation warnings for "legacy behavior" flags
+time: 2024-04-29T14:23:42.804244+02:00
+custom:
+  Author: jtcohen6
+  Issue: "10062"

.github/workflows/main.yml

@@ -135,7 +135,7 @@ jobs:
       - name: generate include
         id: generate-include
         run: |
-          INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' )
+          INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' )
           INCLUDE_GROUPS="["
           for include in ${INCLUDE[@]}; do
               for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do

.github/workflows/schema-check.yml

@@ -13,20 +13,18 @@
 name: Artifact Schema Check
 
 on:
+  pull_request:
+    types: [ opened, reopened, labeled, unlabeled, synchronize ]
+    paths-ignore: [ '.changes/**', '.github/**', 'tests/**', '**.md', '**.yml' ]
+
   workflow_dispatch:
-  pull_request: #TODO: remove before merging
-  push:
-    branches:
-      - "develop"
-      - "*.latest"
-      - "releases/*"
 
 # no special access is needed
 permissions: read-all
 
 env:
   LATEST_SCHEMA_PATH: ${{ github.workspace }}/new_schemas
-  SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}/schema_schanges.txt
+  SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}/schema_changes.txt
   DBT_REPO_DIRECTORY: ${{ github.workspace }}/dbt
   SCHEMA_REPO_DIRECTORY: ${{ github.workspace }}/schemas.getdbt.com
 
@@ -46,15 +44,32 @@ jobs:
         with:
             path: ${{ env.DBT_REPO_DIRECTORY }}
 
+      - name: Check for changes in core/dbt/artifacts
+        # https://github.com/marketplace/actions/paths-changes-filter
+        uses: dorny/paths-filter@v3
+        id: check_artifact_changes
+        with:
+          filters: |
+            artifacts_changed:
+              - 'core/dbt/artifacts/**'
+          list-files: shell
+          working-directory: ${{ env.DBT_REPO_DIRECTORY }}
+
+      - name: Succeed if no artifacts have changed
+        if: steps.check_artifact_changes.outputs.artifacts_changed == 'false'
+        run: |
+          echo "No artifact changes found in core/dbt/artifacts. CI check passed."
+
       - name: Checkout schemas.getdbt.com repo
+        if: steps.check_artifact_changes.outputs.artifacts_changed == 'true'
         uses: actions/checkout@v4
         with:
           repository: dbt-labs/schemas.getdbt.com
           ref: 'main'
-          ssh-key: ${{ secrets.SCHEMA_SSH_PRIVATE_KEY }}
           path: ${{ env.SCHEMA_REPO_DIRECTORY }}
 
       - name: Generate current schema
+        if: steps.check_artifact_changes.outputs.artifacts_changed == 'true'
         run: |
           cd ${{ env.DBT_REPO_DIRECTORY }}
           python3 -m venv env
@@ -65,26 +80,17 @@ jobs:
 
       # Copy generated schema files into the schemas.getdbt.com repo
       # Do a git diff to find any changes
-      # Ignore any date or version changes though
+      # Ignore any lines with date-like (yyyy-mm-dd) or version-like (x.y.z) changes
       - name: Compare schemas
+        if: steps.check_artifact_changes.outputs.artifacts_changed == 'true'
         run: |
           cp -r ${{ env.LATEST_SCHEMA_PATH }}/dbt ${{ env.SCHEMA_REPO_DIRECTORY }}
           cd ${{ env.SCHEMA_REPO_DIRECTORY }}
-          diff_results=$(git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \
-          -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' --compact-summary)
-          if [[ $(echo diff_results) ]]; then
-            echo $diff_results
-            echo "Schema changes detected!"
-            git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \
-              -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' > ${{ env.SCHEMA_DIFF_ARTIFACT }}
-            exit 1
-          else
-            echo "No schema changes detected"
-          fi
+          git diff -I='*[0-9]{4}-[0-9]{2}-[0-9]{2}' -I='*[0-9]+\.[0-9]+\.[0-9]+' --exit-code > ${{ env.SCHEMA_DIFF_ARTIFACT }}
 
       - name: Upload schema diff
         uses: actions/upload-artifact@v4
-        if: ${{ failure() }}
+        if: ${{ failure() && steps.check_artifact_changes.outputs.artifacts_changed == 'true' }}
         with:
-          name: 'schema_schanges.txt'
+          name: 'schema_changes.txt'
           path: '${{ env.SCHEMA_DIFF_ARTIFACT }}'

.github/workflows/test-repeater.yml

@@ -36,7 +36,7 @@ on:
         type: choice
         options:
           - 'ubuntu-latest'
-          - 'macos-latest'
+          - 'macos-12'
           - 'windows-latest'
       num_runs_per_batch:
         description: 'Max number of times to run the test per batch.  We always run 10 batches.'
@@ -101,7 +101,7 @@ jobs:
 
         # mac and windows don't use make due to limitations with docker with those runners in GitHub
       - name: "Set up postgres (macos)"
-        if: inputs.os == 'macos-latest'
+        if: inputs.os == 'macos-12'
         uses: ./.github/actions/setup-postgres-macos
 
       - name: "Set up postgres (windows)"

core/dbt/artifacts/schemas/run/v5/run.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 
 
-from dbt.constants import SECRET_ENV_PREFIX
+from dbt_common.constants import SECRET_ENV_PREFIX
 from dbt.artifacts.resources import CompiledResource
 from dbt.artifacts.schemas.base import (
     BaseArtifactMetadata,

core/dbt/cli/main.py

@@ -170,6 +170,7 @@ def cli(ctx, **kwargs):
 @cli.command("build")
 @click.pass_context
 @global_flags
+@p.empty
 @p.exclude
 @p.export_saved_queries
 @p.full_refresh

core/dbt/cli/option_types.py

@@ -1,10 +1,9 @@
 from click import ParamType, Choice
 
-from dbt.config.utils import parse_cli_yaml_string
+from dbt.config.utils import parse_cli_yaml_string, exclusive_primary_alt_value_setting
 from dbt.events import ALL_EVENT_NAMES
 from dbt.exceptions import ValidationError, OptionNotYamlDictError
 from dbt_common.exceptions import DbtValidationError
-
 from dbt_common.helper_types import WarnErrorOptions
 
 
@@ -52,10 +51,17 @@ class WarnErrorOptionsType(YAML):
     def convert(self, value, param, ctx):
         # this function is being used by param in click
         include_exclude = super().convert(value, param, ctx)
+        exclusive_primary_alt_value_setting(
+            include_exclude, "include", "error", "warn_error_options"
+        )
+        exclusive_primary_alt_value_setting(
+            include_exclude, "exclude", "warn", "warn_error_options"
+        )
 
         return WarnErrorOptions(
             include=include_exclude.get("include", []),
             exclude=include_exclude.get("exclude", []),
+            silence=include_exclude.get("silence", []),
             valid_error_names=ALL_EVENT_NAMES,
         )
 

core/dbt/cli/requires.py

@@ -1,9 +1,11 @@
 import os
 
-import dbt.tracking
-from dbt_common.context import set_invocation_context
+from dbt_common.context import get_invocation_context, set_invocation_context
+from dbt_common.record import Recorder, RecorderMode, get_record_mode_from_env
+from dbt_common.clients.system import get_env
 from dbt_common.invocation import reset_invocation_id
 
+import dbt.tracking
 from dbt.version import installed as installed_version
 from dbt.adapters.factory import adapter_management, register_adapter, get_adapter
 from dbt.context.providers import generate_runtime_macro_context
@@ -44,6 +46,7 @@
 import importlib.util
 import time
 import traceback
+from typing import Optional
 
 
 def preflight(func):
@@ -52,7 +55,14 @@ def wrapper(*args, **kwargs):
         assert isinstance(ctx, Context)
         ctx.obj = ctx.obj or {}
 
-        set_invocation_context(os.environ)
+        set_invocation_context({})
+
+        # Record/Replay
+        setup_record_replay()
+
+        # Must be set after record/replay is set up so that the env can be
+        # recorded or replayed if needed.
+        get_invocation_context()._env = get_env()
 
         # Flags
         flags = Flags(ctx)
@@ -93,6 +103,28 @@ def wrapper(*args, **kwargs):
     return update_wrapper(wrapper, func)
 
 
+def setup_record_replay():
+    rec_mode = get_record_mode_from_env()
+
+    recorder: Optional[Recorder] = None
+    if rec_mode == RecorderMode.REPLAY:
+        recording_path = os.environ["DBT_REPLAY"]
+        recorder = Recorder(RecorderMode.REPLAY, recording_path)
+    elif rec_mode == RecorderMode.RECORD:
+        recorder = Recorder(RecorderMode.RECORD)
+
+    get_invocation_context().recorder = recorder
+
+
+def tear_down_record_replay():
+    recorder = get_invocation_context().recorder
+    if recorder is not None:
+        if recorder.mode == RecorderMode.RECORD:
+            recorder.write("recording.json")
+        elif recorder.mode == RecorderMode.REPLAY:
+            recorder.write_diffs("replay_diffs.json")
+
+
 def postflight(func):
     """The decorator that handles all exception handling for the click commands.
     This decorator must be used before any other decorators that may throw an exception."""
@@ -146,6 +178,8 @@ def wrapper(*args, **kwargs):
                 )
             )
 
+            tear_down_record_replay()
+
         if not success:
             raise ResultExit(result)
 

core/dbt/config/project.py

@@ -27,6 +27,7 @@
 from dbt.adapters.contracts.connection import QueryComment
 from dbt.exceptions import (
     DbtProjectError,
+    DbtExclusivePropertyUseError,
     ProjectContractBrokenError,
     ProjectContractError,
     DbtRuntimeError,
@@ -39,6 +40,7 @@
 from dbt.utils import MultiDict, md5, coerce_dict_str
 from dbt.node_types import NodeType
 from dbt.config.selectors import SelectorDict
+from dbt.config.utils import exclusive_primary_alt_value_setting
 from dbt.contracts.project import (
     Project as ProjectContract,
     SemverString,
@@ -835,10 +837,21 @@ def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags:
             project_flags = profile_project_flags
 
         if project_flags is not None:
+            # handle collapsing `include` and `error` as well as collapsing `exclude` and `warn`
+            # for warn_error_options
+            warn_error_options = project_flags.get("warn_error_options")
+            exclusive_primary_alt_value_setting(
+                warn_error_options, "include", "error", "warn_error_options"
+            )
+            exclusive_primary_alt_value_setting(
+                warn_error_options, "exclude", "warn", "warn_error_options"
+            )
+
             ProjectFlags.validate(project_flags)
             return ProjectFlags.from_dict(project_flags)
-    except (DbtProjectError) as exc:
-        # We don't want to eat the DbtProjectError for UserConfig to ProjectFlags
+    except (DbtProjectError, DbtExclusivePropertyUseError) as exc:
+        # We don't want to eat the DbtProjectError for UserConfig to ProjectFlags or
+        # DbtConfigError for warn_error_options munging
         raise exc
     except (DbtRuntimeError, ValidationError):
         pass

core/dbt/config/renderer.py

@@ -4,9 +4,10 @@
 
 from dbt.clients.jinja import get_rendered
 from dbt_common.clients.jinja import catch_jinja
-from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME
+from dbt_common.constants import SECRET_ENV_PREFIX
+from dbt.constants import DEPENDENCIES_FILE_NAME, SECRET_PLACEHOLDER
 from dbt.context.target import TargetContext
-from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
+from dbt.context.secret import SecretContext
 from dbt.context.base import BaseContext
 from dbt.adapters.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError