merge main

dbt-labs · Dec 5, 2024 · 41bc4a4 · 41bc4a4
2 parents f9eae6f + 6c41bed
commit 41bc4a4
Show file tree

Hide file tree

Showing 43 changed files with 1,045 additions and 282 deletions.
diff --git a/.changes/1.10.3.md b/.changes/1.10.3.md
@@ -0,0 +1 @@
+## dbt-adapters 1.10.3 - October 29, 2024
diff --git a/.changes/1.10.4.md b/.changes/1.10.4.md
@@ -0,0 +1 @@
+## dbt-adapters 1.10.4 - November 11, 2024
diff --git a/.changes/1.11.0.md b/.changes/1.11.0.md
@@ -0,0 +1,12 @@
+## dbt-adapters 1.11.0 - November 11, 2024
+
+### Features
+
+- Use a behavior flag to gate microbatch functionality (instead of an environment variable) ([#327](https://github.com/dbt-labs/dbt-adapters/issues/327))
+
+### Under the Hood
+
+- Add `query_id` to SQLQueryStatus ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
+
+### Contributors
+- [@cmcarthur](https://github.com/cmcarthur) ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md
@@ -0,0 +1,5 @@
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))
diff --git a/.changes/1.7.2.md b/.changes/1.7.2.md
@@ -0,0 +1,16 @@
+## dbt-adapters 1.7.2 - October 21, 2024
+
+### Breaking Changes
+
+- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332))
+
+### Features
+
+- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+### Fixes
+
+- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330))
+
+### Contributors
+- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md
@@ -0,0 +1,9 @@
+## dbt-adapters 1.8.0 - October 29, 2024
+
+### Fixes
+
+- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338))
+
+### Under the Hood
+
+- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301))
diff --git a/.changes/1.9.0.md b/.changes/1.9.0.md
@@ -0,0 +1,6 @@
+## dbt-adapters 1.9.0 - November 13, 2024
+
+### Fixes
+
+- Negate the check for microbatch behavior flag in determining builtins ([#349](https://github.com/dbt-labs/dbt-adapters/issues/349))
+- Move require_batched_execution_for_custom_microbatch_strategy flag to global ([#351](https://github.com/dbt-labs/dbt-adapters/issues/351))
diff --git a/.changes/unreleased/Features-20241104-120653.yaml b/.changes/unreleased/Features-20241104-120653.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add new hard_deletes="new_record" mode for snapshots.
+time: 2024-11-04T12:06:53.225939-05:00
+custom:
+  Author: peterallenwebb
+  Issue: "317"
diff --git a/.changes/unreleased/Features-20241120-112806.yaml b/.changes/unreleased/Features-20241120-112806.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Introduce new Capability for MicrobatchConcurrency support
+time: 2024-11-20T11:28:06.258507-05:00
+custom:
+  Author: michelleark
+  Issue: "359"
diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml
@@ -24,6 +24,6 @@ runs:
       shell: bash
 
     - name: Publish artifacts to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      uses: pypa/gh-action-pypi-publish@release/v1.11
       with:
         repository-url: ${{ inputs.repository-url }}
diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml
@@ -5,7 +5,7 @@ inputs:
     description: File type for file name stub (e.g. "unit-tests")
     required: true
   python-version:
-    description: Python version for the file name stub (e.g. "3.8")
+    description: Python version for the file name stub (e.g. "3.9")
     required: true
   source-file:
     description: File to be uploaded

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out repository

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,19 +18,19 @@ repos:
     -   id: dbt-core-in-adapters-check
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.8.0
     hooks:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311
+        -   --target-version=py312
         -   --force-exclude=dbt/adapters/events/adapter_types_pb2.py
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
         exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/
@@ -41,7 +41,7 @@ repos:
         -   --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.11.2
     hooks:
     -   id: mypy
         exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,18 +5,75 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
-## dbt-adapters 1.10.2 - October 01, 2024
+## dbt-adapters 1.11.0 - November 11, 2024
+
+### Features
+
+- Use a behavior flag to gate microbatch functionality (instead of an environment variable) ([#327](https://github.com/dbt-labs/dbt-adapters/issues/327))
 
 ### Under the Hood
 
-- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))
+- Add `query_id` to SQLQueryStatus ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
 
+### Contributors
+- [@cmcarthur](https://github.com/cmcarthur) ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
 
+## dbt-adapters 1.10.4 - November 11, 2024
+
+## dbt-adapters 1.10.3 - October 29, 2024
+
+## dbt-adapters 1.10.2 - October 01, 2024
+
+### Under the Hood
+
+- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))
 
 ## dbt-adapters 1.10.1 - September 16, 2024
 
 ## dbt-adapters 1.10.0 - September 12, 2024
 
+## dbt-adapters 1.9.0 - November 13, 2024
+
+### Fixes
+
+- Negate the check for microbatch behavior flag in determining builtins ([#349](https://github.com/dbt-labs/dbt-adapters/issues/349))
+- Move require_batched_execution_for_custom_microbatch_strategy flag to global ([#351](https://github.com/dbt-labs/dbt-adapters/issues/351))
+
+
+
+## dbt-adapters 1.8.0 - October 29, 2024
+
+### Fixes
+
+- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338))
+
+### Under the Hood
+
+- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301))
+
+## dbt-adapters 1.7.2 - October 21, 2024
+
+### Breaking Changes
+
+- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332))
+
+### Features
+
+- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+### Fixes
+
+- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330))
+
+### Contributors
+- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))
+
 ## dbt-adapters 1.7.0 - September 19, 2024
 
 ### Features

diff --git a/README.md b/README.md
@@ -2,17 +2,19 @@
   <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
 </p>
 
-# dbt-tests-adapter
+# dbt-adapters
 
 This package is responsible for:
 
 - defining database connection methods
 - caching information from databases
 - determining how relations are defined
 
-There are two major adapter types: base and sql
+In this repo there is also our testing suite used for tesing adapter functionality
+
+# Adapters
 
-# Directories
+There are two major adapter types: base and sql
 
 ## `base`
 

diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py
@@ -1 +1 @@
-version = "1.10.2"
+version = "1.10.4"
diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/files.py b/dbt-tests-adapter/dbt/tests/adapter/basic/files.py
@@ -186,6 +186,10 @@
   {{ config(materialized="incremental") }}
 """
 
+config_materialized_incremental_invalid_strategy = """
+  {{ config(materialized="incremental", incremental_strategy="bad_strategy") }}
+"""
+
 config_materialized_var = """
   {{ config(materialized=var("materialized_var", "table"))}}
 """
@@ -217,3 +221,6 @@
 ephemeral_view_sql = config_materialized_view + model_ephemeral
 ephemeral_table_sql = config_materialized_table + model_ephemeral
 incremental_sql = config_materialized_incremental + model_incremental
+incremental_invalid_strategy_sql = (
+    config_materialized_incremental_invalid_strategy + model_incremental
+)
diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py
@@ -86,6 +86,45 @@ def test_incremental_not_schema_change(self, project):
         assert run_result == RunStatus.Success
 
 
+class BaseIncrementalBadStrategy:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"name": "incremental"}
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "incremental.sql": files.incremental_invalid_strategy_sql,
+            "schema.yml": files.schema_base_yml,
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {"base.csv": files.seeds_base_csv, "added.csv": files.seeds_added_csv}
+
+    @pytest.fixture(autouse=True)
+    def clean_up(self, project):
+        yield
+        with project.adapter.connection_named("__test"):
+            relation = project.adapter.Relation.create(
+                database=project.database, schema=project.test_schema
+            )
+            project.adapter.drop_schema(relation)
+
+    def test_incremental_invalid_strategy(self, project):
+        # seed command
+        results = run_dbt(["seed"])
+        assert len(results) == 2
+
+        # try to run the incremental model, it should fail on the first attempt
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results.results) == 1
+        assert (
+            'dbt could not find an incremental strategy macro with the name "get_incremental_bad_strategy_sql"'
+            in results.results[0].message
+        )
+
+
 class Testincremental(BaseIncremental):
     pass
 

diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py
@@ -1,6 +1,4 @@
-import os
 from pprint import pformat
-from unittest import mock
 
 import pytest
 
@@ -63,7 +61,6 @@ def assert_row_count(self, project, relation_name: str, expected_row_count: int)
 
         assert len(result) == expected_row_count, f"{relation_name}:{pformat(result)}"
 
-    @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
     def test_run_with_event_time(self, project, insert_two_rows_sql):
         # initial run -- backfills all data
         with patch_microbatch_end_time("2020-01-03 13:57:00"):