From b2cae173ed09552845a61bf53c474247e5924a2c Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 16 Jul 2024 15:32:25 -0500 Subject: [PATCH 1/8] pull in new basclass and run test, and update local version of macro for add_column for column quoting --- dbt/include/spark/macros/adapters.sql | 2 +- dev-requirements.txt | 2 +- .../adapter/incremental/test_incremental_on_schema_change.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index a6404a2de..615f4156d 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -422,7 +422,7 @@ {% if add_columns %} add columns {% endif %} {% for column in add_columns %} - {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {{ adapter.quote(column.name )}} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} {%- endset -%} diff --git a/dev-requirements.txt b/dev-requirements.txt index f86fb5ef4..8d8ccd6ba 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter # dev diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index 478329668..7e210951d 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -4,6 +4,7 @@ from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup, + BaseIncrementalCaseSenstivityOnSchemaChange, ) @@ -80,3 +81,7 @@ def test_run_incremental_append_new_columns(self, project): def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_all_columns(project) self.run_incremental_sync_remove_only(project) + + +class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): + pass From fbaaffa2307f12b4d7b26f3a013a7204f3e18c8f Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 10:02:20 -0500 Subject: [PATCH 2/8] update dev-requirements --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8d8ccd6ba..3a6319de6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,7 +3,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapter # dev ipdb~=0.13.13 From cd5d75336b7b7cb412120aacb409abed09a8f08d Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 10:19:03 -0500 Subject: [PATCH 3/8] add changelog --- .changes/unreleased/Under the Hood-20240717-101810.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240717-101810.yaml diff --git a/.changes/unreleased/Under the Hood-20240717-101810.yaml b/.changes/unreleased/Under the Hood-20240717-101810.yaml new file mode 100644 index 000000000..6b2ffb778 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240717-101810.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: update alter_relation_add_remove_columns to use adapter.quote and inherit new test +time: 2024-07-17T10:18:10.178584-05:00 +custom: + Author: McKnight-42 + Issue: "854" From 4fab33ed0955e53115c29657f51a1a23410e6d5e Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 24 Jul 2024 15:50:18 -0500 Subject: [PATCH 4/8] git test passing locally --- .../test_incremental_on_schema_change.py | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index 7e210951d..faea34372 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -1,10 +1,9 @@ import pytest -from dbt.tests.util import run_dbt +from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup, - BaseIncrementalCaseSenstivityOnSchemaChange, ) @@ -83,5 +82,57 @@ def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_remove_only(project) -class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): - pass +_MODELS__SRC_ARTISTS = """ +{{ + config( + materialized='table', + ) +}} +g +{% if var("version", 0) == 0 %} + + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name + +{% else %} + + -- add a non-zero version to the end of the command to get a different version: + -- --vars "{'version': 1}" + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name, 'engineer' as `Job` + +{% endif %} +""" + +_MODELS__DIM_ARTISTS = """ +{{ + config( + materialized='incremental', + on_schema_change='append_new_columns', + ) +}} + +select * from {{ ref("src_artists") }} +""" + + +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestIncrementalCaseSenstivityOnSchemaChange: + @pytest.fixture(scope="class") + def models(self): + return { + "src_artists.sql": _MODELS__SRC_ARTISTS, + "dim_artists.sql": _MODELS__DIM_ARTISTS, + } + + def test_run_incremental_check_quoting_on_new_columns(self, project): + select = "src_artists dim_artists" + run_dbt(["run", "--models", select, "--full-refresh"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) + assert "Job" not in logs + + run_dbt(["run", "--vars", "{'version': 1}"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) + assert "Job" in logs From d5ed1cc4aaf8b294456c6acf5a8c2db384ac0ac3 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 24 Jul 2024 16:41:37 -0500 Subject: [PATCH 5/8] remove extra char --- .../adapter/incremental/test_incremental_on_schema_change.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index faea34372..a38fd3910 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -88,7 +88,6 @@ def test_run_incremental_sync_all_columns(self, project): materialized='table', ) }} -g {% if var("version", 0) == 0 %} select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name From bb5efcdde7cbaf345516c3268633ac9219b9f7af Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 1 Aug 2024 15:54:18 -0500 Subject: [PATCH 6/8] migrate away from spark specific version of test to universal one test --- .../test_incremental_on_schema_change.py | 56 +------------------ 1 file changed, 3 insertions(+), 53 deletions(-) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index a38fd3910..216209867 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -4,6 +4,7 @@ from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup, + BaseIncrementalCaseSenstivityOnSchemaChange, ) @@ -82,56 +83,5 @@ def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_remove_only(project) -_MODELS__SRC_ARTISTS = """ -{{ - config( - materialized='table', - ) -}} -{% if var("version", 0) == 0 %} - - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name - -{% else %} - - -- add a non-zero version to the end of the command to get a different version: - -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name, 'engineer' as `Job` - -{% endif %} -""" - -_MODELS__DIM_ARTISTS = """ -{{ - config( - materialized='incremental', - on_schema_change='append_new_columns', - ) -}} - -select * from {{ ref("src_artists") }} -""" - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestIncrementalCaseSenstivityOnSchemaChange: - @pytest.fixture(scope="class") - def models(self): - return { - "src_artists.sql": _MODELS__SRC_ARTISTS, - "dim_artists.sql": _MODELS__DIM_ARTISTS, - } - - def test_run_incremental_check_quoting_on_new_columns(self, project): - select = "src_artists dim_artists" - run_dbt(["run", "--models", select, "--full-refresh"]) - res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] - ) - assert "Job" not in logs - - run_dbt(["run", "--vars", "{'version': 1}"]) - res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] - ) - assert "Job" in logs +class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): + pass From 1db7911f82b2c2ec275e741c18a09cbe1d064108 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Fri, 2 Aug 2024 14:27:07 -0500 Subject: [PATCH 7/8] add skips for test --- .../adapter/incremental/test_incremental_on_schema_change.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index 216209867..d492e6e75 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -83,5 +83,6 @@ def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_remove_only(project) +@pytest.mark.skip_profile("apache_spark", "spark_session") class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): pass From ea2fe5580d28783e21fdd46ff642a6934a19f35f Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 6 Aug 2024 11:12:26 -0500 Subject: [PATCH 8/8] remove unused import --- .../adapter/incremental/test_incremental_on_schema_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index d492e6e75..9589d9f01 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -1,6 +1,6 @@ import pytest -from dbt.tests.util import run_dbt, run_dbt_and_capture +from dbt.tests.util import run_dbt from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup,