From 3f11435c790808548fa4bcf8ef95da3e0bb5e509 Mon Sep 17 00:00:00 2001 From: artc95 Date: Fri, 2 Feb 2024 17:43:31 +0100 Subject: [PATCH 01/20] switch dependency from dbt-sqlserver to dbt-fabric --- CHANGELOG.md | 5 +++++ dbt/adapters/synapse/__init__.py | 2 +- dbt/adapters/synapse/__version__.py | 2 +- dbt/adapters/synapse/synapse_adapter.py | 4 ++-- dbt/adapters/synapse/synapse_connection_manager.py | 4 ++-- dbt/adapters/synapse/synapse_credentials.py | 4 ++-- setup.py | 6 +++--- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202eee12..9781eeaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## v1.7.0 + +#### Under the hood +* Switch dependency from dbt-sqlserver to dbt-fabric + ## v1.4.0 #### Features diff --git a/dbt/adapters/synapse/__init__.py b/dbt/adapters/synapse/__init__.py index 9f2548ac..6a0d96fd 100644 --- a/dbt/adapters/synapse/__init__.py +++ b/dbt/adapters/synapse/__init__.py @@ -9,7 +9,7 @@ adapter=SynapseAdapter, credentials=SynapseCredentials, include_path=synapse.PACKAGE_PATH, - dependencies=["sqlserver"], + dependencies=["fabric"], ) __all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"] diff --git a/dbt/adapters/synapse/__version__.py b/dbt/adapters/synapse/__version__.py index d619c757..a55413d1 100644 --- a/dbt/adapters/synapse/__version__.py +++ b/dbt/adapters/synapse/__version__.py @@ -1 +1 @@ -version = "1.4.0" +version = "1.7.0" diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index a1608e3f..fcccde7b 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,7 +1,7 @@ -from dbt.adapters.sqlserver import SQLServerAdapter +from dbt.adapters.fabric import FabricAdapter from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager -class SynapseAdapter(SQLServerAdapter): +class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager diff --git a/dbt/adapters/synapse/synapse_connection_manager.py b/dbt/adapters/synapse/synapse_connection_manager.py index d4d0a1d6..23103d1b 100644 --- a/dbt/adapters/synapse/synapse_connection_manager.py +++ b/dbt/adapters/synapse/synapse_connection_manager.py @@ -1,6 +1,6 @@ -from dbt.adapters.sqlserver import SQLServerConnectionManager +from dbt.adapters.fabric import FabricConnectionManager -class SynapseConnectionManager(SQLServerConnectionManager): +class SynapseConnectionManager(FabricConnectionManager): TYPE = "synapse" TOKEN = None diff --git a/dbt/adapters/synapse/synapse_credentials.py b/dbt/adapters/synapse/synapse_credentials.py index c17ea579..43f57a12 100644 --- a/dbt/adapters/synapse/synapse_credentials.py +++ b/dbt/adapters/synapse/synapse_credentials.py @@ -1,10 +1,10 @@ from dataclasses import dataclass -from dbt.adapters.sqlserver import SQLServerCredentials +from dbt.adapters.fabric import FabricCredentials @dataclass -class SynapseCredentials(SQLServerCredentials): +class SynapseCredentials(FabricCredentials): @property def type(self): return "synapse" diff --git a/setup.py b/setup.py index b82646ad..9a756d36 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.4" -dbt_sqlserver_requirement = "dbt-sqlserver~=1.4.0" +dbt_version = "1.7" +dbt_fabric_requirement = "dbt-fabric~=1.7.0" description = """An Azure Synapse adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -73,7 +73,7 @@ def run(self): url="https://github.com/dbt-msft/dbt-synapse", packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, - install_requires=[dbt_sqlserver_requirement], + install_requires=[dbt_fabric_requirement], cmdclass={ "verify": VerifyVersionCommand, }, From cd3b6afca49f8d7727aee95728c17068aa13c2a5 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 10:13:55 +0100 Subject: [PATCH 02/20] fix indexes.sql for "USE statement not supported" error - switch dependency --- .../synapse/macros/adapters/indexes.sql | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/indexes.sql b/dbt/include/synapse/macros/adapters/indexes.sql index 27cdb8b7..a1af97a9 100644 --- a/dbt/include/synapse/macros/adapters/indexes.sql +++ b/dbt/include/synapse/macros/adapters/indexes.sql @@ -126,3 +126,35 @@ declare @drop_remaining_indexes_last nvarchar(max) = ( {% macro create_nonclustered_index(columns, includes=False) %} {{ return(create_nonclustered_index(columns, includes=False)) }} {% endmacro %} + + +{% macro drop_fk_indexes_on_table(relation) -%} + {% call statement('find_references', fetch_result=true) %} + SELECT obj.name AS FK_NAME, + sch.name AS [schema_name], + tab1.name AS [table], + col1.name AS [column], + tab2.name AS [referenced_table], + col2.name AS [referenced_column] + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' + {% endcall %} + {% set references = load_result('find_references')['data'] %} + {% for reference in references -%} + {% call statement('main') -%} + alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] + {%- endcall %} + {% endfor %} +{% endmacro %} From 26e284961da42e779bcbd127278482ee77997c0f Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 11:01:17 +0100 Subject: [PATCH 03/20] add snapshot.sql for "Distribution option must be explicitly specified in 'CREATE TABLE AS SELECT'..." error - switch dependency --- .../materializations/snapshots/snapshot.sql | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 dbt/include/synapse/macros/materializations/snapshots/snapshot.sql diff --git a/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql b/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql new file mode 100644 index 00000000..effe87c3 --- /dev/null +++ b/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql @@ -0,0 +1,61 @@ +{% macro synapse__create_columns(relation, columns) %} + {# default__ macro uses "add column" + TSQL preferes just "add" + #} + + {% set columns %} + {% for column in columns %} + , CAST(NULL AS {{column.data_type}}) AS {{column_name}} + {% endfor %} + {% endset %} + + {% set tempTableName %} + [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}_{{ range(1300, 19000) | random }}] + {% endset %} + + {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%} + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + {% set tempTable %} + CREATE TABLE {{tempTableName}} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} + {% endset %} + + {% call statement('create_temp_table') -%} + {{ tempTable }} + {%- endcall %} + + {% set dropTable %} + DROP TABLE [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] + {% endset %} + + {% call statement('drop_table') -%} + {{ dropTable }} + {%- endcall %} + + {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%} + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + {% set createTable %} + CREATE TABLE {{ relation }} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} + {% endset %} + + {% call statement('create_Table') -%} + {{ createTable }} + {%- endcall %} + + {% set dropTempTable %} + DROP TABLE {{tempTableName}} + {% endset %} + + {% call statement('drop_temp_table') -%} + {{ dropTempTable }} + {%- endcall %} +{% endmacro %} From 33ce51474ae337abba95bcbac5bd1740c3cc47d1 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 11:24:04 +0100 Subject: [PATCH 04/20] add "drop_schema" macro in schema.sql, without Synapse-unsupported "IF EXISTS" - switch dependency --- .../synapse/macros/adapters/schema.sql | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/schema.sql b/dbt/include/synapse/macros/adapters/schema.sql index d24dd20e..fa472091 100644 --- a/dbt/include/synapse/macros/adapters/schema.sql +++ b/dbt/include/synapse/macros/adapters/schema.sql @@ -6,3 +6,23 @@ END {% endcall %} {% endmacro %} + +{% macro synapse__drop_schema(relation) -%} + {%- set relations_in_schema = list_relations_without_caching(relation) %} + + {% for row in relations_in_schema %} + {%- set schema_relation = api.Relation.create(database=relation.database, + schema=relation.schema, + identifier=row[1], + type=row[3] + ) -%} + {% do drop_relation(schema_relation) %} + {%- endfor %} + + {% call statement('drop_schema') -%} + IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}') + BEGIN + EXEC('DROP SCHEMA [{{ relation.without_identifier().schema }}]') + END + {% endcall %} +{% endmacro %} From 3bc4d979ab944641aef9bafb816eb4c076266eb7 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 12:37:49 +0100 Subject: [PATCH 05/20] skip ephemeral and changing_relation_type tests; change time(stamp) column type to datetime2 --- tests/functional/adapter/test_basic.py | 1 + tests/functional/adapter/test_changing_relation_type.py | 2 ++ tests/functional/adapter/test_data_types.py | 2 +- tests/functional/adapter/test_docs.py | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index d676a2ec..87133424 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -32,6 +32,7 @@ class TestEmptySynapse(BaseEmpty): pass +@pytest.mark.skip(reason="ephemeral not supported") class TestEphemeralSynapse(BaseEphemeral): pass diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index e80da716..e56f7f44 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,5 +1,7 @@ +import pytest from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator +@pytest.mark.skip(reason="CTAS is not supported without a underlying table definition.") class TestChangeRelationTypesSynapse(BaseChangeRelationTypeValidator): pass diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_data_types.py index 7181ab31..b65f735b 100644 --- a/tests/functional/adapter/test_data_types.py +++ b/tests/functional/adapter/test_data_types.py @@ -46,7 +46,7 @@ def seeds(self): - name: expected config: column_types: - timestamp_col: "datetimeoffset" + timestamp_col: "datetime2" """ return { diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index e8fc4296..7ea26be4 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -22,7 +22,7 @@ def expected_catalog(self, project): role="dbo", id_type="int", text_type="varchar", - time_type="datetime", + time_type="datetime2", view_type="VIEW", table_type="BASE TABLE", model_stats=no_stats(), @@ -37,7 +37,7 @@ def expected_catalog(self, project): role="dbo", id_type="int", text_type="varchar", - time_type="datetime", + time_type="datetime2", bigint_type="int", view_type="VIEW", table_type="BASE TABLE", From cf7b5cffae2e64c1ea999f36f0326c7dcd5f8b0d Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 12:25:51 +0100 Subject: [PATCH 06/20] update "rename_relation" macro in relation.sql, fix "new name already in use...duplicate...not permitted" - switch dependency --- dbt/include/synapse/macros/adapters/relation.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dbt/include/synapse/macros/adapters/relation.sql b/dbt/include/synapse/macros/adapters/relation.sql index 7f3c6a94..be43be08 100644 --- a/dbt/include/synapse/macros/adapters/relation.sql +++ b/dbt/include/synapse/macros/adapters/relation.sql @@ -20,7 +20,17 @@ {% macro synapse__rename_relation(from_relation, to_relation) -%} {% call statement('rename_relation') -%} + -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted" + if object_id ('{{ to_relation.include(database=False) }}','V') is not null + begin + drop view {{ to_relation.include(database=False) }} + end + + if object_id ('{{ to_relation.include(database=False) }}','U') is not null + begin + drop table {{ to_relation.include(database=False) }} + end - rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} + rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} {%- endcall %} {% endmacro %} From 946a521aaedc986fec98632320b720350a030797 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 12:32:31 +0100 Subject: [PATCH 07/20] do not skip changing_relation_type test, succeeds after "rename_relation" macro in relation.sql updated - switch dependency --- tests/functional/adapter/test_changing_relation_type.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index e56f7f44..e80da716 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,7 +1,5 @@ -import pytest from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator -@pytest.mark.skip(reason="CTAS is not supported without a underlying table definition.") class TestChangeRelationTypesSynapse(BaseChangeRelationTypeValidator): pass From 40e1737c879b31e60e112071fc4fa4ddebe67bd0 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 14:13:42 +0100 Subject: [PATCH 08/20] add truncate_relation macro in relation.sql, to overwrite fabric's macro that does not specify Distribution option in "CREATE TABLE AS SELECT" - switch dependency --- dbt/include/synapse/macros/adapters/relation.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/relation.sql b/dbt/include/synapse/macros/adapters/relation.sql index be43be08..8c1d3582 100644 --- a/dbt/include/synapse/macros/adapters/relation.sql +++ b/dbt/include/synapse/macros/adapters/relation.sql @@ -34,3 +34,9 @@ rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} {%- endcall %} {% endmacro %} + +{% macro synapse__truncate_relation(relation) %} + {% call statement('truncate_relation') -%} + truncate table {{ relation }} + {%- endcall %} +{% endmacro %} From eed57dc3e81a10797bbdd8d52ccb21ae53d006fe Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 15:07:03 +0100 Subject: [PATCH 09/20] overwrite assertion of "grant " in log_output to fix test_grants; link to dependency switch discussion in CHANGELOG.md - switch dependency --- CHANGELOG.md | 2 +- tests/functional/adapter/test_grants.py | 45 +++++++++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9781eeaa..b4abdbad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v1.7.0 #### Under the hood -* Switch dependency from dbt-sqlserver to dbt-fabric +* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) ## v1.4.0 diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 1eb193e0..026512d0 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -1,9 +1,12 @@ -import pytest from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants -from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants +from dbt.tests.adapter.grants.test_snapshot_grants import ( + BaseSnapshotGrants, + user2_snapshot_schema_yml, +) +from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file class TestIncrementalGrantsSynapse(BaseIncrementalGrants): @@ -27,11 +30,33 @@ class TestSeedGrantsSynapse(BaseSeedGrants): class TestSnapshotGrantsSynapse(BaseSnapshotGrants): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - # ('42000', '[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server] - # Merge statements with a WHEN NOT MATCHED [BY TARGET] clause must - # target a hash distributed table. (100087) (SQLExecDirectW)') - "snapshots": {"test": {"dist": "HASH(id)", "index": "HEAP"}}, - } + def test_snapshot_grants(self, project, get_test_users): + test_users = get_test_users + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # run the snapshot + results = run_dbt(["snapshot"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + snapshot_id = "snapshot.test.my_snapshot" + snapshot = manifest.nodes[snapshot_id] + expected = {select_privilege_name: [test_users[0]]} + assert snapshot.config.grants == expected + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) + + # run it again, nothing should have changed + # since dbt selects into temporary table, drops existing, selects into original table name, + # SELECT needs to be granted again, so "grant " expected in log_output! + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " in log_output # grant expected + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) + + # change the grantee, assert it updates + updated_yaml = self.interpolate_name_overrides(user2_snapshot_schema_yml) + write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + expected = {select_privilege_name: [test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) From b644ed8bec46b5c82b7e97b3e9c4d6cee23dbb02 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 15:24:22 +0100 Subject: [PATCH 10/20] update test_timestamps.py to expect datetime2 - switch dependency --- tests/functional/adapter/test_timestamps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index 91c72f18..106e7905 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -15,4 +15,4 @@ def expected_schema(self): @pytest.fixture(scope="class") def expected_sql(self): - return '''select SYSDATETIME() as "current_timestamp"''' + return '''select CAST( SYSDATETIME() AS DATETIME2(6) ) as "current_timestamp"''' From 049a799b46a71aedcff57cc3994b29021720081c Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 7 Feb 2024 16:07:15 +0100 Subject: [PATCH 11/20] fix versioning and reference to adhere to 1.4 --- CHANGELOG.md | 2 +- dbt/adapters/synapse/__version__.py | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4abdbad..7e3773db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## v1.7.0 +## v1.4.1rc1 #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) diff --git a/dbt/adapters/synapse/__version__.py b/dbt/adapters/synapse/__version__.py index a55413d1..9b363e24 100644 --- a/dbt/adapters/synapse/__version__.py +++ b/dbt/adapters/synapse/__version__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.4.1rc1" diff --git a/setup.py b/setup.py index 9a756d36..7cf2fe88 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.7" -dbt_fabric_requirement = "dbt-fabric~=1.7.0" +dbt_version = "1.4.1rc1" +dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3" description = """An Azure Synapse adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) From 9185ef602a0e0280bf3bdf9da6a908b838b27918 Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 7 Feb 2024 16:09:34 +0100 Subject: [PATCH 12/20] add pyodbc binary override for ARM --- dev_requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index b4833c9b..02e93bd0 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,7 +4,8 @@ wheel==0.40.0 pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.4.5 +dbt-tests-adapter~=1.4.9 +pyodbc==4.0.39 --no-binary :all: aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From ede60e4033076db8a9ed3604d529803c5dc57097 Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 7 Feb 2024 16:55:39 +0100 Subject: [PATCH 13/20] overwrite "create_schema" method in SynapseAdapter, to use added macro "synapse__create_schema_with_authorization", fixed failing test_schema.py by adding "schema_authorization" variable to dbt_profile --- dbt/adapters/synapse/synapse_adapter.py | 20 +++++++++++++ .../synapse/macros/adapters/schema.sql | 9 ++++++ tests/functional/adapter/test_schema.py | 29 +++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index fcccde7b..5c02b579 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,7 +1,27 @@ +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.fabric import FabricAdapter +from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME +from dbt.events.functions import fire_event +from dbt.events.types import SchemaCreation from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager + + def create_schema(self, relation: BaseRelation) -> None: + relation = relation.without_identifier() + fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) + macro_name = CREATE_SCHEMA_MACRO_NAME + kwargs = { + "relation": relation, + } + + if self.config.credentials.schema_authorization: + kwargs["schema_authorization"] = self.config.credentials.schema_authorization + macro_name = "synapse__create_schema_with_authorization" + + self.execute_macro(macro_name, kwargs=kwargs) + self.commit_if_has_connection() diff --git a/dbt/include/synapse/macros/adapters/schema.sql b/dbt/include/synapse/macros/adapters/schema.sql index fa472091..106496fb 100644 --- a/dbt/include/synapse/macros/adapters/schema.sql +++ b/dbt/include/synapse/macros/adapters/schema.sql @@ -7,6 +7,15 @@ {% endcall %} {% endmacro %} +{% macro synapse__create_schema_with_authorization(relation, schema_authorization) -%} + {% call statement('create_schema') -%} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}') + BEGIN + EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]') + END + {% endcall %} +{% endmacro %} + {% macro synapse__drop_schema(relation) -%} {%- set relations_in_schema = list_relations_without_caching(relation) %} diff --git a/tests/functional/adapter/test_schema.py b/tests/functional/adapter/test_schema.py index 90570610..c283555d 100644 --- a/tests/functional/adapter/test_schema.py +++ b/tests/functional/adapter/test_schema.py @@ -1,9 +1,33 @@ import os import pytest +from conftest import _profile_ci_azure_auto, _profile_user, _profile_user_azure from dbt.tests.util import run_dbt +@pytest.fixture(scope="class") +def dbt_profile_target(request): + profile = request.config.getoption("--profile") + + if profile == "ci_azure_auto": + return { + **_profile_ci_azure_auto(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + if profile == "user": + return { + **_profile_user(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + if profile == "user_azure": + return { + **_profile_user_azure(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + + raise ValueError(f"Unknown profile: {profile}") + + class TestSchemaCreation: @pytest.fixture(scope="class") def models(self): @@ -14,11 +38,6 @@ def models(self): """, } - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - @staticmethod def _verify_schema_owner(schema_name, owner, project): get_schema_owner = f""" From 34adcc5556fefb8b93c9c881fae2fa533041d709 Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 7 Feb 2024 17:45:10 +0100 Subject: [PATCH 14/20] adjust dbt_version in setup.py to 1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7cf2fe88..dec7e0c8 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.4.1rc1" +dbt_version = "1.4" dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3" description = """An Azure Synapse adapter plugin for dbt""" From a46b5867c04609ecb30f5847b8fd71f49a705ae4 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 12 Feb 2024 14:05:52 +0100 Subject: [PATCH 15/20] fix synapse_adapter.py to import _make_ref_key_msg (instead of ..._dict, not available in dbt-core v1.4.9, see https://github.com/dbt-labs/dbt-core/compare/v1.4.9...v1.7.6) --- dbt/adapters/synapse/synapse_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index 5c02b579..9e6dfddb 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,5 +1,5 @@ from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.cache import _make_ref_key_dict +from dbt.adapters.cache import _make_ref_key_msg from dbt.adapters.fabric import FabricAdapter from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME from dbt.events.functions import fire_event @@ -13,7 +13,7 @@ class SynapseAdapter(FabricAdapter): def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) + fire_event(SchemaCreation(relation=_make_ref_key_msg(relation))) macro_name = CREATE_SCHEMA_MACRO_NAME kwargs = { "relation": relation, From 7fadfffd313ca4c14a141813d9b1b7924645bd3e Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 12 Feb 2024 15:20:47 +0100 Subject: [PATCH 16/20] alert to remove pyodbc from dev_requirements.txt, if using Windows --- CHANGELOG.md | 1 + CONTRIBUTING.md | 2 +- dev_requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3773db..193369a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) + * if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt ## v1.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd5ccd52..fb39952f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ pyenv activate dbt-synapse Install the development dependencies and pre-commit and get information about possible make commands: ```shell -make dev +make dev # if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt make help ``` diff --git a/dev_requirements.txt b/dev_requirements.txt index 02e93bd0..06071e8d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,7 @@ pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.4.9 -pyodbc==4.0.39 --no-binary :all: +pyodbc==4.0.39 --no-binary :all: # if Windows, remove aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From 5fd1851c59f824f00a61f958d303bdd7c0e9ed3e Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 14 Feb 2024 09:52:15 +0100 Subject: [PATCH 17/20] remove pyodbc from dev_requirements.txt, alert about pyodbc for Mac users --- CHANGELOG.md | 3 ++- CONTRIBUTING.md | 2 +- dev_requirements.txt | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 193369a6..6e1b2e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) - * if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt + * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt + * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle ## v1.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb39952f..eca250d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ pyenv activate dbt-synapse Install the development dependencies and pre-commit and get information about possible make commands: ```shell -make dev # if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt +make dev # for Mac users, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt before running `make dev` make help ``` diff --git a/dev_requirements.txt b/dev_requirements.txt index 06071e8d..7e77f77b 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,6 @@ pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.4.9 -pyodbc==4.0.39 --no-binary :all: # if Windows, remove aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From b8792d9aee02e0ad5e90d8ec9f028a2f1b4c1211 Mon Sep 17 00:00:00 2001 From: "Arthur Chionh (dataroots)" <90323829+arthurcht@users.noreply.github.com> Date: Mon, 19 Feb 2024 20:50:19 +0100 Subject: [PATCH 18/20] add permissions block in workflow .yml --- .github/workflows/integration-tests-azure.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1e3e76b4..8e7e00c7 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -10,6 +10,15 @@ on: # yamllint disable-line rule:truthy - master jobs: + build-me: + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + deployments: read + packages: none + pull-requests: write + security-events: write integration-tests-azure: name: Integration tests on Azure strategy: From 151327467ad8dc1bd1cb36c6501c9223132c63d5 Mon Sep 17 00:00:00 2001 From: "Arthur Chionh (dataroots)" <90323829+arthurcht@users.noreply.github.com> Date: Mon, 19 Feb 2024 20:53:03 +0100 Subject: [PATCH 19/20] update permissions block in workflow .yml --- .github/workflows/integration-tests-azure.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 8e7e00c7..2decb350 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -10,7 +10,12 @@ on: # yamllint disable-line rule:truthy - master jobs: - build-me: + integration-tests-azure: + name: Integration tests on Azure + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + msodbc_version: ["17", "18"] runs-on: ubuntu-latest permissions: actions: read @@ -19,13 +24,6 @@ jobs: packages: none pull-requests: write security-events: write - integration-tests-azure: - name: Integration tests on Azure - strategy: - matrix: - python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - msodbc_version: ["17", "18"] - runs-on: ubuntu-latest container: image: ghcr.io/dbt-msft/dbt-sqlserver:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: From 9a58d9cb9ebbf23cff688cd837a40ae24d0b1e21 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 20 Feb 2024 09:29:16 +0100 Subject: [PATCH 20/20] commit datarootsio/dependency_dbtsqlserver_to_dbtfabric --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1b2e67..082e56f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## v1.4.1rc1 #### Under the hood -* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) +* Switch dependency from dbt-sqlserver to dbt-fabric [#441](https://github.com/dbt-msft/dbt-sqlserver/issues/441) * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt - * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle + * [Stackoverflow](https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle) about pyodbc "Symbol not found: _SQLAllocHandle" error ## v1.4.0