diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 18fd7fa..48c4b9f 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d641e2c..f300748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +### V1.8.7 +* Improving table materialization to minimize downtime #189 +* Handling temp tables in incremental models #188 +* Add label support to filter queries #181 +* Addressed bug - incremental models cannot full refresh #179 +* Addressed bug - #197, dbt test incorrect syntax with macro helpers.sql + ### v1.8.0rc2 ## Bug Fixes diff --git a/dbt/adapters/fabric/__version__.py b/dbt/adapters/fabric/__version__.py index fd11d27..9675f25 100644 --- a/dbt/adapters/fabric/__version__.py +++ b/dbt/adapters/fabric/__version__.py @@ -1 +1 @@ -version = "1.8.6" +version = "1.8.7" diff --git a/dbt/include/fabric/macros/adapters/catalog.sql b/dbt/include/fabric/macros/adapters/catalog.sql index cd02a29..6bd1398 100644 --- a/dbt/include/fabric/macros/adapters/catalog.sql +++ b/dbt/include/fabric/macros/adapters/catalog.sql @@ -1,131 +1,132 @@ {% macro fabric__get_catalog(information_schemas, schemas) -%} + {% set query_label = apply_label() %} + {%- call statement('catalog', fetch_result=True) -%} - {%- call statement('catalog', fetch_result=True) -%} - - with - principals as ( - select - name as principal_name, - principal_id as principal_id - from - sys.database_principals {{ information_schema_hints() }} - ), - - schemas as ( - select - name as schema_name, - schema_id as schema_id, - principal_id as principal_id - from - sys.schemas {{ information_schema_hints() }} - ), - - tables as ( - select - object_id, - name as table_name, - schema_id as schema_id, - principal_id as principal_id, - 'BASE TABLE' as table_type - from - sys.tables {{ information_schema_hints() }} - ), - - tables_with_metadata as ( - select - object_id, - table_name, - schema_name, - coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, - table_type - from - tables - join schemas on tables.schema_id = schemas.schema_id - ), - - views as ( - select - object_id, - name as table_name, - schema_id as schema_id, - principal_id as principal_id, - 'VIEW' as table_type - from - sys.views {{ information_schema_hints() }} - ), - - views_with_metadata as ( - select - object_id, - table_name, - schema_name, - coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, - table_type - from - views - join schemas on views.schema_id = schemas.schema_id - ), - - tables_and_views as ( - select - object_id, - table_name, - schema_name, - principal_name, - table_type - from - tables_with_metadata - join principals on tables_with_metadata.owner_principal_id = principals.principal_id - union all - select - object_id, - table_name, - schema_name, - principal_name, - table_type - from - views_with_metadata - join principals on views_with_metadata.owner_principal_id = principals.principal_id - ), - - cols as ( + with + principals as ( + select + name as principal_name, + principal_id as principal_id + from + sys.database_principals {{ information_schema_hints() }} + ), + + schemas as ( + select + name as schema_name, + schema_id as schema_id, + principal_id as principal_id + from + sys.schemas {{ information_schema_hints() }} + ), + + tables as ( + select + object_id, + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'BASE TABLE' as table_type + from + sys.tables {{ information_schema_hints() }} + ), + + tables_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + tables + join schemas on tables.schema_id = schemas.schema_id + ), + + views as ( + select + object_id, + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'VIEW' as table_type + from + sys.views {{ information_schema_hints() }} + ), + + views_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + views + join schemas on views.schema_id = schemas.schema_id + ), + + tables_and_views as ( + select + object_id, + table_name, + schema_name, + principal_name, + table_type + from + tables_with_metadata + join principals on tables_with_metadata.owner_principal_id = principals.principal_id + union all + select + object_id, + table_name, + schema_name, + principal_name, + table_type + from + views_with_metadata + join principals on views_with_metadata.owner_principal_id = principals.principal_id + ), + + cols as ( + + select + c.object_id, + c.name as column_name, + c.column_id as column_index, + t.name as column_type + from sys.columns as c {{ information_schema_hints() }} + left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} + ) select - c.object_id, - c.name as column_name, - c.column_id as column_index, - t.name as column_type - from sys.columns as c {{ information_schema_hints() }} - left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} - ) - - select - DB_NAME() as table_database, - tv.schema_name as table_schema, - tv.table_name, - tv.table_type, - null as table_comment, - tv.principal_name as table_owner, - cols.column_name, - cols.column_index, - cols.column_type, - null as column_comment - from tables_and_views tv - join cols on tv.object_id = cols.object_id - where ({%- for schema in schemas -%} - upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%}) - - order by column_index - - {%- endcall -%} - - {{ return(load_result('catalog').table) }} + DB_NAME() as table_database, + tv.schema_name as table_schema, + tv.table_name, + tv.table_type, + null as table_comment, + tv.principal_name as table_owner, + cols.column_name, + cols.column_index, + cols.column_type, + null as column_comment + from tables_and_views tv + join cols on tv.object_id = cols.object_id + where ({%- for schema in schemas -%} + upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%}) + + order by column_index + {{ query_label }} + + {%- endcall -%} + + {{ return(load_result('catalog').table) }} {%- endmacro %} {% macro fabric__get_catalog_relations(information_schema, relations) -%} - + {% set query_label = apply_label() %} {%- call statement('catalog', fetch_result=True) -%} with @@ -260,7 +261,7 @@ ) order by column_index - + {{ query_label }} {%- endcall -%} {{ return(load_result('catalog').table) }} diff --git a/dbt/include/fabric/macros/adapters/columns.sql b/dbt/include/fabric/macros/adapters/columns.sql index 69f2ce8..26c8fc2 100644 --- a/dbt/include/fabric/macros/adapters/columns.sql +++ b/dbt/include/fabric/macros/adapters/columns.sql @@ -11,42 +11,46 @@ {% endmacro %} {% macro fabric__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} + {% set query_label = apply_label() %} + {% call statement('get_columns_in_relation', fetch_result=True) %} + + with mapping as ( + select + row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position, + c.name collate database_default as column_name, + t.name as data_type, + c.max_length as character_maximum_length, + c.precision as numeric_precision, + c.scale as numeric_scale + from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }} + inner join sys.types t {{ information_schema_hints() }} + on c.user_type_id = t.user_type_id + where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}') + ) - with mapping as ( select - row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position, - c.name collate database_default as column_name, - t.name as data_type, - c.max_length as character_maximum_length, - c.precision as numeric_precision, - c.scale as numeric_scale - from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }} - inner join sys.types t {{ information_schema_hints() }} - on c.user_type_id = t.user_type_id - where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}') - ) - - select - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - from mapping - order by ordinal_position - - {% endcall %} - {% set table = load_result('get_columns_in_relation').table %} - {{ return(sql_convert_columns_in_relation(table)) }} + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale + from mapping + order by ordinal_position + {{ query_label }} + + {% endcall %} + {% set table = load_result('get_columns_in_relation').table %} + {{ return(sql_convert_columns_in_relation(table)) }} {% endmacro %} {% macro fabric__get_columns_in_query(select_sql) %} + {% set query_label = apply_label() %} {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} select TOP 0 * from ( {{ select_sql }} ) as __dbt_sbq where 0 = 1 + {{ query_label }} {% endcall %} {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} @@ -84,6 +88,7 @@ {% set tempTable %} CREATE TABLE {{tempTableName}} AS SELECT {{query_result_text}}, CAST({{ column_name }} AS {{new_column_type}}) AS {{column_name}} FROM {{ relation.schema }}.{{ relation.identifier }} + {{ apply_label() }} {% endset %} {% call statement('create_temp_table') -%} @@ -100,7 +105,7 @@ {% set createTable %} CREATE TABLE {{ relation.schema }}.{{ relation.identifier }} - AS SELECT * FROM {{tempTableName}} + AS SELECT * FROM {{tempTableName}} {{ apply_label() }} {% endset %} {% call statement('create_Table') -%} diff --git a/dbt/include/fabric/macros/adapters/metadata.sql b/dbt/include/fabric/macros/adapters/metadata.sql index 2fe6a58..4d4f9fa 100644 --- a/dbt/include/fabric/macros/adapters/metadata.sql +++ b/dbt/include/fabric/macros/adapters/metadata.sql @@ -1,3 +1,10 @@ + +{% macro apply_label() %} + {{ log (config.get('query_tag','dbt-fabric'))}} + {%- set query_label = config.get('query_tag','dbt-fabric-dw') -%} + OPTION (LABEL = '{{query_label}}'); +{% endmacro %} + {% macro information_schema_hints() %} {{ return(adapter.dispatch('information_schema_hints')()) }} {% endmacro %} @@ -20,15 +27,14 @@ {% macro fabric__list_schemas(database) %} {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%} select name as [schema] - from sys.schemas {{ information_schema_hints() }} + from sys.schemas {{ information_schema_hints() }} {{ apply_label() }} {% endcall %} {{ return(load_result('list_schemas').table) }} {% endmacro %} {% macro fabric__check_schema_exists(information_schema, schema) -%} {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%} - - SELECT count(*) as schema_exist FROM sys.schemas WHERE name = '{{ schema }}' + SELECT count(*) as schema_exist FROM sys.schemas WHERE name = '{{ schema }}' {{ apply_label() }} {%- endcall %} {{ return(load_result('check_schema_exists').table) }} {% endmacro %} @@ -53,6 +59,7 @@ ) select * from base where [schema] like '{{ schema_relation.schema }}' + {{ apply_label() }} {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} @@ -78,6 +85,7 @@ select * from base where [schema] like '{{ schema_relation.schema }}' and [name] like '{{ schema_relation.identifier }}' + {{ apply_label() }} {% endcall %} {{ return(load_result('get_relation_without_caching').table) }} {% endmacro %} @@ -97,6 +105,7 @@ upper(o.name) = upper('{{ relation.identifier }}')){%- if not loop.last %} or {% endif -%} {%- endfor -%} ) + {{ apply_label() }} {%- endcall -%} {{ return(load_result('last_modified')) }} diff --git a/dbt/include/fabric/macros/adapters/relation.sql b/dbt/include/fabric/macros/adapters/relation.sql index a6fe0f0..6eac9bb 100644 --- a/dbt/include/fabric/macros/adapters/relation.sql +++ b/dbt/include/fabric/macros/adapters/relation.sql @@ -1,4 +1,4 @@ -{% macro fabric__make_temp_relation(base_relation, suffix) %} +{% macro fabric__make_temp_relation(base_relation, suffix='__dbt_temp') %} {%- set temp_identifier = base_relation.identifier ~ suffix -%} {%- set temp_relation = base_relation.incorporate( path={"identifier": temp_identifier}) -%} @@ -23,6 +23,7 @@ and refs.referenced_entity_name = '{{ relation.identifier }}' and refs.referencing_class = 1 and obj.type = 'V' + {{ apply_label() }} {% endcall %} {% set references = load_result('find_references')['data'] %} {% for reference in references -%} diff --git a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql index bd045df..bb7d4b3 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql @@ -3,54 +3,52 @@ {%- set full_refresh_mode = (should_full_refresh()) -%} {% set target_relation = this.incorporate(type='table') %} {%- set relation = load_cached_relation(this) -%} - {%- set existing_relation = none %} {% if relation.type == 'table' %} {% set existing_relation = target_relation %} {% elif relation.type == 'view' %} {% set existing_relation = get_or_create_relation(relation.database, relation.schema, relation.identifier, relation.type)[1] %} + {#-- Can't overwrite a view with a table - we must drop --#} + {{ log("Dropping relation " ~ existing_relation ~ " because it is a view and target is a table.") }} + {{ adapter.drop_relation(existing_relation) }} {% endif %} -- configs {%- set unique_key = config.get('unique_key') -%} {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} - {%- set temp_relation = make_temp_relation(target_relation)-%} {% set grant_config = config.get('grants') %} {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} {{ run_hooks(pre_hooks, inside_transaction=True) }} - -- naming a temp relation - {% set tmp_relation_view = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + {% if existing_relation is none or full_refresh_mode or existing_relation.is_view %} - -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation_view) %} + {% set tmp_vw_relation = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_vw_relation) }} + -- Dropping target relation if exists + {{ adapter.drop_relation(target_relation) }} - {% if existing_relation is none %} {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} {%- endcall -%} - {% elif existing_relation.is_view %} - {#-- Can't overwrite a view with a table - we must drop --#} - {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} - {% do adapter.drop_relation(existing_relation) %} + -- Dropping temp view relation + {{ adapter.drop_relation(tmp_vw_relation) }} - {%- call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql)}} - {%- endcall -%} + {% else %} - {% elif full_refresh_mode %} - {%- call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql)}} - {%- endcall -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {{ adapter.drop_relation(temp_relation) }} + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} - {% else %} {%- call statement('create_tmp_relation') -%} {{ get_create_table_as_sql(True, temp_relation, sql)}} {%- endcall -%} + {{ adapter.drop_relation(tmp_tble_vw_relation) }} {% do adapter.expand_target_column_types( from_relation=temp_relation, to_relation=target_relation) %} @@ -67,10 +65,10 @@ {%- call statement('main') -%} {{ strategy_sql_macro_func(strategy_arg_dict) }} {%- endcall -%} + + {{ adapter.drop_relation(temp_relation) }} {% endif %} - {% do adapter.drop_relation(tmp_relation_view) %} - {% do adapter.drop_relation(temp_relation) %} {{ run_hooks(post_hooks, inside_transaction=True) }} {% set target_relation = target_relation.incorporate(type='table') %} {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} diff --git a/dbt/include/fabric/macros/materializations/models/incremental/merge.sql b/dbt/include/fabric/macros/materializations/models/incremental/merge.sql index 6f4315a..b5a0903 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/merge.sql @@ -14,6 +14,8 @@ {% endmacro %} {% macro fabric__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %} + + {% set query_label = apply_label() %} {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} {% if unique_key %} @@ -27,13 +29,13 @@ {{ source }}.{{ key }} = {{ target }}.{{ key }} {{ "and " if not loop.last }} {% endfor %} - ) {% if incremental_predicates %} {% for predicate in incremental_predicates %} and {{ predicate }} {% endfor %} - {% endif %}; + {% endif %} + {{ query_label }} {% else %} delete from {{ target }} where ( @@ -45,7 +47,8 @@ {% for predicate in incremental_predicates %} and {{ predicate }} {% endfor %} - {%- endif -%}; + {%- endif -%} + {{ query_label }} {% endif %} {% endif %} @@ -53,5 +56,5 @@ ( select {{ dest_cols_csv }} from {{ source }} - ) + ){{ query_label }} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index 00c84bd..e2a946a 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,12 +1,13 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} - - {% set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - {{ get_create_view_as_sql(tmp_relation, sql) }} + {% set query_label = apply_label() %} + {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + {% do adapter.drop_relation(tmp_vw_relation) %} + {{ get_create_view_as_sql(tmp_vw_relation, sql) }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} - CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] + CREATE TABLE {{relation}} {{ build_columns_constraints(relation) }} {{ get_assert_columns_equivalent(sql) }} {% set listColumns %} @@ -15,11 +16,11 @@ {% endfor %} {%endset%} - INSERT INTO [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] - ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]; + INSERT INTO {{relation}} ({{listColumns}}) + SELECT {{listColumns}} FROM {{tmp_vw_relation}} {{ query_label }} {%- else %} - EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);'); + {%- set query_label_option = query_label.replace("'", "''") -%} + EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}} {{ query_label_option }}'); {% endif %} - {% do adapter.drop_relation(tmp_relation)%} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index 88f1df9..4eb1f4a 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -4,41 +4,60 @@ {%- set target_relation = this.incorporate(type='table') %} {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%} - {%- set backup_relation = none %} - {% if (existing_relation != none and existing_relation.type == "table") %} - {%- set backup_relation = make_backup_relation(target_relation, 'table') -%} - {% elif (existing_relation != none and existing_relation.type == "view") %} - {%- set backup_relation = make_backup_relation(target_relation, 'view') -%} - {% endif %} - - {% if (existing_relation != none) %} - -- drop the temp relations if they exist already in the database - {% do adapter.drop_relation(backup_relation) %} - -- Rename target relation as backup relation - {{ adapter.rename_relation(existing_relation, backup_relation) }} + {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to + -- downtime, but it should be a relatively infrequent occurrence #} + {% if existing_relation is not none and not existing_relation.is_table %} + {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} + {{ adapter.drop_relation(existing_relation) }} {% endif %} -- grab current tables grants config for comparision later on {% set grant_config = config.get('grants') %} + -- Making a temp relation + {% set temp_relation = make_temp_relation(target_relation, '__dbt_temp') %} + + -- Drop temp relation if it exists before materializing temp relation + {{ adapter.drop_relation(temp_relation) }} + + {% set tmp_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + + {{ adapter.drop_relation(tmp_vw_relation) }} + {{ run_hooks(pre_hooks, inside_transaction=False) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} - -- naming a temp relation - {% set tmp_relation = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - - -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation) %} - -- build model {% call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql) }} - {%- endcall %} + {{ create_table_as(False, temp_relation, sql) }} + {% endcall %} + + {% if existing_relation is not none and existing_relation.is_table %} + + -- making a backup relation, this will come in use when contract is enforced or not + {%- set backup_relation = make_backup_relation(existing_relation, 'table') -%} - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation) %} + -- Dropping a temp relation if it exists + {{ adapter.drop_relation(backup_relation) }} + + -- Rename existing relation to back up relation + {{ adapter.rename_relation(existing_relation, backup_relation) }} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, target_relation) }} + + -- Drop backup relation + {{ adapter.drop_relation(backup_relation) }} + + {%- else %} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, target_relation) }} + + {% endif %} + + {{ adapter.drop_relation(tmp_vw_relation) }} -- cleanup {{ run_hooks(post_hooks, inside_transaction=True) }} @@ -48,13 +67,6 @@ -- `COMMIT` happens here {{ adapter.commit() }} - {% if (backup_relation != none) %} - -- finally, drop the foreign key references if exists - {{ drop_fk_indexes_on_table(backup_relation) }} - -- drop existing/backup relation after the commit - {% do adapter.drop_relation(backup_relation) %} - {% endif %} - -- Add constraints including FK relation. {{ build_model_constraints(target_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} diff --git a/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql b/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql index f0ec12d..cf63055 100644 --- a/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql +++ b/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql @@ -5,7 +5,6 @@ {% macro fabric__create_view_exec(relation, sql) -%} {%- set temp_view_sql = sql.replace("'", "''") -%} - {{ get_use_database_sql(relation.database) }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} diff --git a/dbt/include/fabric/macros/materializations/seeds/helpers.sql b/dbt/include/fabric/macros/materializations/seeds/helpers.sql index 68d5f80..4526ccd 100644 --- a/dbt/include/fabric/macros/materializations/seeds/helpers.sql +++ b/dbt/include/fabric/macros/materializations/seeds/helpers.sql @@ -43,6 +43,7 @@ {%- endfor -%}) {%- if not loop.last%},{%- endif %} {%- endfor %} + {{ apply_label()}} {% endset %} {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index a37ae64..bf7c77a 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -22,7 +22,7 @@ {% set tempTable %} CREATE TABLE {{tempTableName}} - AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} + AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} {{ apply_label() }} {% endset %} {% call statement('create_temp_table') -%} @@ -39,7 +39,7 @@ {% set createTable %} CREATE TABLE {{ relation }} - AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} + AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} {{ apply_label() }} {% endset %} {% call statement('create_Table') -%} @@ -190,10 +190,20 @@ {% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} {% set temp_relation = make_temp_relation(target_relation) %} + {{ adapter.drop_relation(temp_relation) }} + {% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + {% call statement('build_snapshot_staging_relation') %} {{ get_create_table_as_sql(True, temp_relation, select) }} {% endcall %} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + {% do return(temp_relation) %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql index bf874bb..07c6789 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql @@ -44,9 +44,9 @@ {% set tmp_relation_view = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation_view) %} + {{ adapter.drop_relation(tmp_relation_view) }} {% set final_sql = get_create_table_as_sql(False, target_relation, build_sql) %} - {% do adapter.drop_relation(tmp_relation_view) %} + {{ adapter.drop_relation(tmp_relation_view) }} {% else %} @@ -88,7 +88,7 @@ {{ final_sql }} {% endcall %} - {% do adapter.drop_relation(temp_snapshot_relation) %} + {{ adapter.drop_relation(temp_snapshot_relation) }} {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql index 838da43..e2d3e99 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql @@ -1,15 +1,12 @@ {% macro fabric__snapshot_merge_sql(target, source, insert_cols) %} - {%- set insert_cols_csv = insert_cols | join(', ') -%} + {%- set insert_cols_csv = insert_cols | join(', ') -%} {%- set target_table = target.include(database=False) -%} {%- set source_table = source.include(database=False) -%} - {% set target_columns_list = [] %} - {% for column in insert_cols %} {% set target_columns_list = target_columns_list.append("DBT_INTERNAL_SOURCE."+column) %} {% endfor %} - {%- set target_columns = target_columns_list | join(', ') -%} UPDATE DBT_INTERNAL_DEST @@ -18,9 +15,11 @@ INNER JOIN {{ source_table }} as DBT_INTERNAL_SOURCE on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id WHERE DBT_INTERNAL_DEST.dbt_valid_to is null - AND DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete'); + AND DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') + {{ apply_label() }} INSERT INTO {{ target_table }} ({{ insert_cols_csv }}) SELECT {{target_columns}} FROM {{ source_table }} as DBT_INTERNAL_SOURCE - WHERE DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'; + WHERE DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' + {{ apply_label() }} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/tests/helpers.sql b/dbt/include/fabric/macros/materializations/tests/helpers.sql index 9e4b057..4f04547 100644 --- a/dbt/include/fabric/macros/materializations/tests/helpers.sql +++ b/dbt/include/fabric/macros/materializations/tests/helpers.sql @@ -9,7 +9,7 @@ {% if main_sql.strip().lower().startswith('with') %} {% set testview %} - {{ target.schema }}.testview_{{ range(1300, 19000) | random }} + [{{ target.schema }}.testview_{{ range(1300, 19000) | random }}] {% endset %} {% set sql = main_sql.replace("'", "''")%} diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 6759424..9168c0c 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -503,8 +503,11 @@ def test__constraints_ddl(self, project, expected_sql): generated_sql_generic, "foreign_key_model", "" ) generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_cluase = generated_sql_wodb.replace( + "OPTION (LABEL = 'dbt-fabric-dw');", "" + ) assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( - generated_sql_wodb.strip() + generated_sql_option_cluase.strip() ) @@ -568,8 +571,11 @@ def test__model_constraints_ddl(self, project, expected_sql): generated_sql_generic, "foreign_key_model", "" ) generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_cluase = generated_sql_wodb.replace( + "OPTION (LABEL = 'dbt-fabric-dw');", "" + ) assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( - generated_sql_wodb.strip() + generated_sql_option_cluase.strip() ) @@ -618,7 +624,10 @@ def test__constraints_enforcement_rollback( # Verify the previous table still exists relation = relation_from_name(project.adapter, "my_model") - model_backup = str(relation).replace("my_model", "my_model__dbt_backup") + # table materialization does not rename existing relation to back)up relation + # Rather, a new relation is created with __dbt_temp suffix. + # If model creation is successful, then the existing model is renamed as backup and then dropped + model_backup = str(relation).replace("my_model", "my_model") old_model_exists_sql = f"select * from {model_backup}" old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") assert len(old_model_exists) == 1