Merge pull request #210 from microsoft/dbtsynapse1.7_dataroots

v1.7.0rc1

.pre-commit-config.yaml

@@ -18,6 +18,7 @@ repos:
       - id: check-merge-conflict
       - id: no-commit-to-branch
       - id: fix-byte-order-marker
+        exclude: ^tests/functional/adapter/data/seed_bom.csv
       - id: mixed-line-ending
       - id: check-docstring-first
   - repo: 'https://github.com/adrienverge/yamllint'

CHANGELOG.md

@@ -1,4 +1,25 @@
 # Changelog
+## v1.7.0rc1
+
+## Features
+
+* Support for [dbt-core 1.7](https://github.com/dbt-labs/dbt-core/releases/tag/v1.7.0)
+  * implement custom `date_spine` and `generate_series` macro logic for synapse to match nested-CTE limitation
+
+#### Under the hood
+
+  * Decouple `get_catalog` macro (marco override test is skipped as it is not covering the behavior)
+  * Add UTF-8-BOM CSV and exclude from precommit format override
+  * New/extended adapter test zones
+    - get_last_relation_modified
+    - date_spine
+    - generate_series
+    - get_intervals_between
+    - get_powers_of_two
+    - store_test_failures
+    - dbt_clone (same target and state)
+    - seed
+
 ## v1.6.0rc1
 
 * Support for [dbt-core 1.6](https://github.com/dbt-labs/dbt-core/releases/tag/v1.6.0)
@@ -20,8 +41,13 @@
 
 ## v.1.5.0rc1
 
+## Features
+
 * Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0)
   * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros
+
+#### Under the hood
+
   * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class.
   * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse
   * Add adapter tests zones

dbt/adapters/synapse/__version__.py

@@ -1 +1 @@
-version = "1.6.0rc1"
+version = "1.7.0rc1"

dbt/include/synapse/macros/adapters/catalog.sql

@@ -0,0 +1,147 @@
+{% macro synapse__get_catalog_tables_sql(information_schemas) -%}
+-- avoid with statement to be able to wrap into CTE (limitation of Synapse)
+
+    SELECT
+        table_name,
+        table_schema,
+        principal_name,
+        table_type
+    FROM (
+        SELECT
+            table_name,
+            schema_name AS table_schema,
+            COALESCE(relations.principal_id, schemas.principal_id) AS owner_principal_id,
+            table_type
+        FROM (
+            SELECT
+                name AS table_name,
+                schema_id AS schema_id,
+                principal_id AS principal_id,
+                'BASE TABLE' AS table_type
+            FROM
+                sys.tables {{ information_schema_hints() }}
+            UNION ALL
+            SELECT
+                name AS table_name,
+                schema_id AS schema_id,
+                principal_id AS principal_id,
+                'VIEW' AS table_type
+            FROM
+                sys.views {{ information_schema_hints() }}
+        ) AS relations
+        JOIN (
+            SELECT
+                name AS schema_name,
+                schema_id AS schema_id,
+                principal_id AS principal_id
+            FROM
+                sys.schemas {{ information_schema_hints() }}
+        ) AS schemas ON relations.schema_id = schemas.schema_id
+    ) AS relations_with_metadata
+    JOIN (
+        SELECT
+            name AS principal_name,
+            principal_id AS principal_id
+        FROM
+            sys.database_principals {{ information_schema_hints() }}
+    ) AS principals ON relations_with_metadata.owner_principal_id = principals.principal_id
+
+{% endmacro %}
+
+{% macro synapse__get_catalog_columns_sql(information_schemas) -%}
+
+    select
+        table_catalog as table_database,
+        table_schema,
+        table_name,
+        column_name,
+        ordinal_position as column_index,
+        data_type as column_type
+    from INFORMATION_SCHEMA.COLUMNS {{ information_schema_hints() }}
+
+{% endmacro %}
+
+{% macro synapse__get_catalog_schemas_where_clause_sql(schemas) -%}
+    where ({%- for schema in schemas -%}
+        upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+    {%- endfor -%})
+{%- endmacro %}
+
+{% macro synapse__get_catalog_relations_where_clause_sql(relations) %}
+
+    where (
+        {%- for relation in relations -%}
+            {% if relation.schema and relation.identifier %}
+                (
+                    upper(table_schema) = upper('{{ relation.schema }}')
+                    and upper(table_name) = upper('{{ relation.identifier }}')
+                )
+            {% elif relation.schema %}
+                (
+                    upper(table_schema) = upper('{{ relation.schema }}')
+                )
+            {% else %}
+                {% do exceptions.raise_compiler_error(
+                    '`get_catalog_relations` requires a list of relations, each with a schema'
+                ) %}
+            {% endif %}
+
+            {%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+
+{% endmacro %}
+
+{% macro synapse__get_catalog_results_sql() -%}
+    select
+        cols.table_database,
+        tv.table_schema,
+        tv.table_name,
+        tv.table_type,
+        null as table_comment,
+        tv.principal_name as table_owner,
+        cols.column_name,
+        cols.column_index,
+        cols.column_type,
+        null as column_comment
+    from tables as tv
+    join columns as cols on tv.table_schema = cols.table_schema and tv.table_name = cols.table_name
+    order by cols.column_index
+{%- endmacro %}
+
+-- combine everything into the get_catalog_(relations) macro
+{% macro synapse__get_catalog(information_schema, schemas) -%}
+
+    {% set query %}
+        with tables as (
+            {{ synapse__get_catalog_tables_sql(information_schema) }}
+            {{ synapse__get_catalog_schemas_where_clause_sql(schemas) }}
+        ),
+        columns as (
+            {{ synapse__get_catalog_columns_sql(information_schema) }}
+            {{ synapse__get_catalog_schemas_where_clause_sql(schemas) }}
+        )
+        {{ synapse__get_catalog_results_sql() }}
+    {%- endset -%}
+
+    {{ return(run_query(query)) }}
+
+{%- endmacro %}
+
+{% macro synapse__get_catalog_relations(information_schema, relations) -%}
+
+    {% set query %}
+        with tables as (
+            {{ synapse__get_catalog_tables_sql(information_schema) }}
+            {{ synapse__get_catalog_relations_where_clause_sql(relations) }}
+        ),
+        columns as (
+            {{ synapse__get_catalog_columns_sql(information_schema) }}
+            {{ synapse__get_catalog_relations_where_clause_sql(relations) }}
+        )
+        {{ synapse__get_catalog_results_sql() }}
+    {%- endset -%}
+
+    {{ return(run_query(query)) }}
+
+{%- endmacro %}

dbt/include/synapse/macros/adapters/relation.sql

@@ -5,22 +5,26 @@
 {% endmacro %}
 
 {% macro synapse__drop_relation_script(relation) -%}
+  {% if relation is not none %}
   {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
     {% set object_id_type = 'V' %}
   {% elif relation.type == 'table'%}
     {% set object_id_type = 'U' %}
   {%- else -%} invalid target name
   {% endif %}
-
-  if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
-  {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
-    begin
-    drop view {{ relation.include(database=False) }}
-    end
-  {% elif relation.type == 'table' %}
-    begin
-    drop {{ relation.type }} {{ relation.include(database=False) }}
-    end
+    if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
+    {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
+      begin
+      drop view {{ relation.include(database=False) }}
+      end
+    {% elif relation.type == 'table' %}
+      begin
+      drop {{ relation.type }} {{ relation.include(database=False) }}
+      end
+    {% endif %}
+  {% else %}
+    -- no object to drop
+    select 1 as nothing
   {% endif %}
 {% endmacro %}
 

dbt/include/synapse/macros/adapters/replace.sql

@@ -0,0 +1,50 @@
+{% macro get_replace_sql(existing_relation, target_relation, sql) %}
+    {{- log('Applying REPLACE to: ' ~ existing_relation) -}}
+    {{- adapter.dispatch('get_replace_sql', 'synapse')(existing_relation, target_relation, sql) -}}
+{% endmacro %}
+
+
+{% macro synapse__get_replace_sql(existing_relation, target_relation, sql) %}
+
+    {# /* use a create or replace statement if possible */ #}
+
+    {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}
+
+    {% if is_replaceable and existing_relation.is_view %}
+        {{ get_replace_view_sql(target_relation, sql) }}
+
+    {% elif is_replaceable and existing_relation.is_table %}
+        {{ get_replace_table_sql(target_relation, sql) }}
+
+    {% elif is_replaceable and existing_relation.is_materialized_view %}
+        {{ get_replace_materialized_view_sql(target_relation, sql) }}
+
+    {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}
+
+    {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}
+    {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}
+        {{ get_create_intermediate_sql(target_relation, sql) }};
+        {{ get_create_backup_sql(existing_relation) }};
+        {{ get_rename_intermediate_sql(target_relation) }};
+        {{ synapse__drop_relation(existing_relation) }}
+
+    {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}
+    {%- elif target_relation.can_be_renamed -%}
+        {{ get_create_intermediate_sql(target_relation, sql) }};
+        {{ synapse__drop_relation(existing_relation) }};
+        {{ get_rename_intermediate_sql(target_relation) }}
+
+    {# /* create target_relation in place by first backing up the existing relation */ #}
+    {%- elif existing_relation.can_be_renamed -%}
+        {{ get_create_backup_sql(existing_relation) }};
+        {{ get_create_sql(target_relation, sql) }};
+        {{ synapse__drop_relation(existing_relation) }}
+
+    {# /* no renaming is allowed, so just drop and create */ #}
+    {%- else -%}
+        {{ synapse__drop_relation(existing_relation) }};
+        {{ get_create_sql(target_relation, sql) }}
+
+    {%- endif -%}
+
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/materialized_view/create_materialized_view_as.sql

@@ -0,0 +1,30 @@
+{% macro ref(model_name) %}
+
+    {% do return(builtins.ref(model_name).include(database=false)) %}
+
+{% endmacro %}
+
+
+{% macro synapse__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}
+    {# Synapse does not have ALTER...RENAME function, so use synapse__rename_relation_script #}
+
+    {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
+    EXEC('
+    CREATE materialized view [{{intermediate_relation.schema}}].[{{intermediate_relation.identifier}}]
+    WITH ( DISTRIBUTION = {{dist}} )
+    AS {{ sql }}
+    ');
+
+    {{ synapse__rename_relation_script(existing_relation, backup_relation) }}
+    {{ synapse__rename_relation_script(intermediate_relation, relation) }}
+
+{% endmacro %}
+
+{% macro synapse__get_create_materialized_view_as_sql(relation, sql) %}
+    {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
+
+    CREATE materialized view [{{relation.schema}}].[{{relation.identifier}}]
+    WITH ( DISTRIBUTION = {{dist}} )
+    AS {{ sql }}
+
+{% endmacro %}