Merge branch 'dbtsynapse1.6_dataroots' of github.com:microsoft/dbt-sy…

…napse into dbtsynapse1.7_dataroots

.github/workflows/integration-tests-azure.yml

@@ -14,9 +14,16 @@ jobs:
     name: Integration tests on Azure
     strategy:
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
         msodbc_version: ["17", "18"]
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      deployments: read
+      packages: none
+      pull-requests: write
+      security-events: write
     container:
       image: ghcr.io/dbt-msft/dbt-sqlserver:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }}
     steps:

CHANGELOG.md

@@ -14,13 +14,44 @@
     - store_test_failures
     - dbt_clone (same target and state)
     - seed
+## v1.6.0rc1
+
+* Support for [dbt-core 1.6](https://github.com/dbt-labs/dbt-core/releases/tag/v1.6.0)
+
+#### Breaking Changes
+* Dropped support for Python 3.7 ([#7082](https://github.com/dbt-labs/dbt-core/issues/7082https://github.com/dbt-labs/dbt-core/issues/7082))
+
+## Features
+* Add support for materialized views ([#6911](https://github.com/dbt-labs/dbt-core/issues/6911))
+  * important note! unlike [dbt's materialized view](https://docs.getdbt.com/docs/build/materializations), [Synapse's materialized view](https://learn.microsoft.com/en-us/sql/t-sql/statements/create-materialized-view-as-select-transact-sql?view=azure-sqldw-latest&context=%2Fazure%2Fsynapse-analytics%2Fcontext%2Fcontext) must be created using aggregation and/or "GROUP BY"! 
+* ~~dbt clone ([#7258](https://github.com/dbt-labs/dbt-core/issues/7258)~~ Synapse does not support CLONE)
+* Revamp dbt debug ([#7104](https://github.com/dbt-labs/dbt-core/issues/7104))
+* Added new adapter zone tests
+  - constraints
+  - null_compare
+  - validate_sql
+  - equals
+  - dbt_clone
+
+## v.1.5.0rc1
+
+* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0)
+  * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros
+  * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class.
+  * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse
+  * Add adapter tests zones
+    - caching
+    - column_types
+    - constraints
+    - hooks
+    - simple_copy
 
 ## v1.4.1rc1
 
 #### Under the hood
-* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441)
+* Switch dependency from dbt-sqlserver to dbt-fabric [#441](https://github.com/dbt-msft/dbt-sqlserver/issues/441)
   * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt
-  * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle
+  * [Stackoverflow](https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle) about pyodbc "Symbol not found: _SQLAllocHandle" error
 
 ## v1.4.0
 

dbt/adapters/synapse/__init__.py

@@ -1,6 +1,7 @@
 from dbt.adapters.base import AdapterPlugin
 
 from dbt.adapters.synapse.synapse_adapter import SynapseAdapter
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 from dbt.adapters.synapse.synapse_credentials import SynapseCredentials
 from dbt.include import synapse
@@ -12,4 +13,10 @@
     dependencies=["fabric"],
 )
 
-__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"]
+__all__ = [
+    "Plugin",
+    "SynapseConnectionManager",
+    "SynapseColumn",
+    "SynapseAdapter",
+    "SynapseCredentials",
+]

dbt/adapters/synapse/synapse_adapter.py

@@ -1,15 +1,21 @@
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
 from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.cache import _make_ref_key_dict
 from dbt.adapters.fabric import FabricAdapter
 from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType
 from dbt.events.functions import fire_event
 from dbt.events.types import SchemaCreation
 
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 
 
 class SynapseAdapter(FabricAdapter):
     ConnectionManager = SynapseConnectionManager
+    Column = SynapseColumn
 
     def create_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
@@ -25,3 +31,51 @@ def create_schema(self, relation: BaseRelation) -> None:
 
         self.execute_macro(macro_name, kwargs=kwargs)
         self.commit_if_has_connection()
+
+    class ConstraintSupport(str, Enum):
+        ENFORCED = "enforced"
+        NOT_ENFORCED = "not_enforced"
+        NOT_SUPPORTED = "not_supported"
+
+    # https://learn.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-table-constraints#table-constraints
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,  # no CHECK support for Synapse
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,  # no FK support for Synapse
+    }
+
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text.
+        Should be overriden by adapters which need custom constraint rendering."""
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"check {constraint.expression}"
+        elif constraint.type == ConstraintType.not_null:
+            return "not null"
+        elif constraint.type == ConstraintType.unique:
+            return "unique NOT ENFORCED"
+        elif constraint.type == ConstraintType.primary_key:
+            return "primary key NONCLUSTERED NOT ENFORCED"
+        elif constraint.type == ConstraintType.foreign_key:
+            return "foreign key"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return constraint.expression
+        else:
+            return None
+
+    @classmethod
+    def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
+        rendered_column_constraints = []
+
+        for v in raw_columns.values():
+            rendered_column_constraint = [f"[{v['name']}] {v['data_type']}"]
+            for con in v.get("constraints", None):
+                constraint = cls._parse_column_constraint(con)
+                c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
+                if c is not None:
+                    rendered_column_constraint.append(c)
+            rendered_column_constraints.append(" ".join(rendered_column_constraint))
+
+        return rendered_column_constraints

dbt/adapters/synapse/synapse_column.py

@@ -0,0 +1,16 @@
+from dbt.adapters.fabric import FabricColumn
+
+
+class SynapseColumn(FabricColumn):
+    # extending list of integer types for synapse
+    def is_integer(self) -> bool:
+        return self.dtype.lower() in [
+            # real types
+            "smallint",
+            "bigint",
+            "tinyint",
+            "serial",
+            "bigserial",
+            "int",
+            "bit",
+        ]

dbt/include/synapse/macros/adapters/persist_docs.sql

@@ -0,0 +1,4 @@
+{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer
+  https://github.com/dbt-msft/dbt-sqlserver/issues/134
+  https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16
+ #}

dbt/include/synapse/macros/adapters/relation.sql

@@ -5,21 +5,35 @@
 {% endmacro %}
 
 {% macro synapse__drop_relation_script(relation) -%}
-  {% if relation.type == 'view' -%}
+  {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
     {% set object_id_type = 'V' %}
   {% elif relation.type == 'table'%}
     {% set object_id_type = 'U' %}
   {%- else -%} invalid target name
   {% endif %}
+
   if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
+  {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
+    begin
+    drop view {{ relation.include(database=False) }}
+    end
+  {% elif relation.type == 'table' %}
     begin
     drop {{ relation.type }} {{ relation.include(database=False) }}
     end
+  {% endif %}
 {% endmacro %}
 
-
 {% macro synapse__rename_relation(from_relation, to_relation) -%}
-  {% call statement('rename_relation') -%}
+  {# dbt needs this 'call' macro, but it overwrites other SQL when reused in other macros #}
+  {# so '_script' macro is reuseable script, for other macros to combine with more SQL #}
+
+  {% call statement('rename_relation') %}
+    {{ synapse__rename_relation_script(from_relation, to_relation) }}
+  {%- endcall %}
+{% endmacro %}
+
+{% macro synapse__rename_relation_script(from_relation, to_relation) -%}
   -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted"
   if object_id ('{{ to_relation.include(database=False) }}','V') is not null
     begin
@@ -32,11 +46,10 @@
     end
 
   rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }}
-  {%- endcall %}
 {% endmacro %}
 
 {% macro synapse__truncate_relation(relation) %}
     {% call statement('truncate_relation') -%}
-        truncate table {{ relation }}
+      truncate table {{ relation }}
     {%- endcall %}
 {% endmacro %}

dbt/include/synapse/macros/adapters/show.sql

@@ -0,0 +1,22 @@
+{% macro get_show_sql(compiled_code, sql_header, limit) -%}
+  {%- if sql_header -%}
+  {{ sql_header }}
+  {%- endif -%}
+  {%- if limit is not none -%}
+  {{ get_limit_subquery_sql(compiled_code, limit) }}
+  {%- else -%}
+  {{ compiled_code }}
+  {%- endif -%}
+{% endmacro %}
+
+{% macro get_limit_subquery_sql(sql, limit) %}
+  {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}
+{% endmacro %}
+
+{# Synapse doesnt support ANSI LIMIT clause #}
+{% macro synapse__get_limit_subquery_sql(sql, limit) %}
+    select top {{ limit }} *
+    from (
+        {{ sql }}
+    ) as model_limit_subq
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/materialized_view/materialized_view.sql

@@ -0,0 +1,29 @@
+{% macro ref(model_name) %}
+
+    {% do return(builtins.ref(model_name).include(database=false)) %}
+
+{% endmacro %}
+
+{% macro synapse__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}
+    {# Synapse does not have ALTER...RENAME function, so use synapse__rename_relation_script #}
+
+    {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
+    EXEC('
+    CREATE materialized view {{ intermediate_relation.include(database=False) }}
+    WITH ( DISTRIBUTION = {{dist}} )
+    AS {{ sql }}
+    ');
+
+    {{ synapse__rename_relation_script(existing_relation, backup_relation) }}
+    {{ synapse__rename_relation_script(intermediate_relation, relation) }}
+
+{% endmacro %}
+
+{% macro synapse__get_create_materialized_view_as_sql(relation, sql) %}
+    {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
+
+    CREATE materialized view {{ relation.include(database=False) }}
+    WITH ( DISTRIBUTION = {{dist}} )
+    AS {{ sql }}
+
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/table/create_table_as.sql

@@ -10,16 +10,35 @@
 
    {{ synapse__drop_relation_script(relation) }}
 
-   EXEC('create view [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}] as
-    {{ temp_view_sql }}
-    ');
-
-  CREATE TABLE {{ relation.include(database=False) }}
-    WITH(
-      DISTRIBUTION = {{dist}},
-      {{index}}
-      )
-    AS (SELECT * FROM [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}])
+   {{ synapse__create_view_as(tmp_relation, sql) }}
+
+   {% set contract_config = config.get('contract') %}
+
+   {% if contract_config.enforced %}
+
+        {{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
+
+        CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}]
+        {{ synapse__build_columns_constraints(tmp_relation) }}
+        WITH(
+          DISTRIBUTION = {{dist}},
+          {{index}}
+        )
+        {{ get_assert_columns_equivalent(sql)  }}
+
+        {% set listColumns %}
+            {% for column in model['columns'] %}
+                {{ "["~column~"]" }}{{ ", " if not loop.last }}
+            {% endfor %}
+        {%endset%}
+        {{ synapse__build_model_constraints(relation) }}
+
+        INSERT INTO [{{relation.schema}}].[{{relation.identifier}}]
+        ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]
+
+    {%- else %}
+        EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);');
+    {% endif %}
 
    {{ synapse__drop_relation_script(tmp_relation) }}
 

dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql

@@ -0,0 +1,17 @@
+{% macro synapse__build_columns_constraints(relation) %}
+  {# loop through user_provided_columns to create DDL with data types and constraints #}
+    {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}
+    (
+      {% for c in raw_column_constraints -%}
+        {{ c }}{{ "," if not loop.last }}
+      {% endfor %}
+    )
+{% endmacro %}
+
+{% macro synapse__build_model_constraints(relation) %}
+  {# loop through user_provided_columns to create DDL with data types and constraints #}
+    {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}
+    {% for c in raw_model_constraints -%}
+        alter table {{ relation.include(database=False) }} {{c}};
+    {% endfor -%}
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/view/create_view_as.sql

@@ -1,4 +1,15 @@
 {% macro synapse__create_view_as(relation, sql) -%}
-  create view {{ relation.include(database=False) }} as
-    {{ sql }}
+
+  {%- set temp_view_sql = sql.replace("'", "''") -%}
+
+  {% set contract_config = config.get('contract') %}
+
+    {{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
+
+  {% if contract_config.enforced %}
+      {{ get_assert_columns_equivalent(sql) }}
+  {%- endif %}
+
+  EXEC('create view {{ relation.include(database=False) }} as {{ temp_view_sql }};');
+
 {% endmacro %}

dbt/include/synapse/macros/utils/split_part.sql

@@ -2,8 +2,18 @@
     We have to replace the macro from dbt-sqlserver since that one uses XML which is an unsupported data type in Synapse.
     The function below is not supported in Synapse Dedicated SQL according to the documentation, but it seems to work.
 #}
+
 {% macro synapse__split_part(string_text, delimiter_text, part_number) %}
 
-    (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) where ordinal = {{ part_number }})
+    {% if part_number >= 0 %}
+
+        (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) where ordinal = {{ part_number }})
+
+    {% else %}
+
+        (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1)
+        where ordinal = len(replace({{ string_text }}, {{delimiter_text}}, '')) + 1 + {{ part_number }})
+
+    {% endif %}
 
 {% endmacro %}

setup.py

@@ -83,7 +83,6 @@ def run(self):
         "Operating System :: Microsoft :: Windows",
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",