From 36187bf956625f31877c8fdaf7aea4cc3a3b22fd Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Wed, 22 May 2024 23:08:07 -0700 Subject: [PATCH] changes to drop_relation_if_exists --- .../fabric/macros/adapters/relation.sql | 52 ++++++++----------- dbt/include/fabric/macros/adapters/schema.sql | 2 +- .../models/incremental/incremental.sql | 20 ++++--- .../materializations/models/table/clone.sql | 2 +- .../materializations/models/table/table.sql | 4 +- .../materializations/models/view/view.sql | 4 +- .../materializations/snapshots/helpers.sql | 2 +- tests/functional/adapter/test_caching.py | 2 + 8 files changed, 45 insertions(+), 43 deletions(-) diff --git a/dbt/include/fabric/macros/adapters/relation.sql b/dbt/include/fabric/macros/adapters/relation.sql index 677c847..4899c95 100644 --- a/dbt/include/fabric/macros/adapters/relation.sql +++ b/dbt/include/fabric/macros/adapters/relation.sql @@ -7,45 +7,39 @@ {% endmacro %} {% macro fabric__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - {{ fabric__drop_relation_script(relation) }} - {%- endcall %} -{% endmacro %} - -{% macro fabric__drop_relation_script(relation) -%} - - {% if relation.type == 'view' -%} + {% if relation.type == 'view' -%} {% call statement('find_references', fetch_result=true) %} - {{ get_use_database_sql(relation.database) }} - select - sch.name as schema_name, - obj.name as view_name - from sys.sql_expression_dependencies refs - inner join sys.objects obj - on refs.referencing_id = obj.object_id - inner join sys.schemas sch - on obj.schema_id = sch.schema_id - where refs.referenced_database_name = '{{ relation.database }}' - and refs.referenced_schema_name = '{{ relation.schema }}' - and refs.referenced_entity_name = '{{ relation.identifier }}' - and refs.referencing_class = 1 - and obj.type = 'V' + {{ get_use_database_sql(relation.database) }} + select + sch.name as schema_name, + obj.name as view_name + from sys.sql_expression_dependencies refs + inner join sys.objects obj + on refs.referencing_id = obj.object_id + inner join sys.schemas sch + on obj.schema_id = sch.schema_id + where refs.referenced_database_name = '{{ relation.database }}' + and refs.referenced_schema_name = '{{ relation.schema }}' + and refs.referenced_entity_name = '{{ relation.identifier }}' + and refs.referencing_class = 1 + and obj.type = 'V' {% endcall %} {% set references = load_result('find_references')['data'] %} {% for reference in references -%} - -- dropping referenced view {{ reference[0] }}.{{ reference[1] }} - {{ fabric__drop_relation_script(relation.incorporate( - type="view", - path={"schema": reference[0], "identifier": reference[1]})) }} + -- dropping referenced view {{ reference[0] }}.{{ reference[1] }} + {{ drop_relation_if_exists(relation.incorporate( + type="view", + path={"schema": reference[0], "identifier": reference[1]})) }} {% endfor %} + {{ get_use_database_sql(relation.database) }} + EXEC('DROP VIEW IF EXISTS {{ relation.include(database=False) }};'); {% elif relation.type == 'table'%} {% set object_id_type = 'U' %} - + {{ get_use_database_sql(relation.database) }} + EXEC('DROP TABLE IF EXISTS {{ relation.include(database=False) }};'); {%- else -%} {{ exceptions.raise_not_implemented('Invalid relation being dropped: ' ~ relation) }} {% endif %} - {{ get_use_database_sql(relation.database) }} - EXEC('DROP {{ relation.type }} IF EXISTS {{ relation.include(database=False) }};'); {% endmacro %} {% macro fabric__rename_relation(from_relation, to_relation) -%} diff --git a/dbt/include/fabric/macros/adapters/schema.sql b/dbt/include/fabric/macros/adapters/schema.sql index 099ba59..f85c056 100644 --- a/dbt/include/fabric/macros/adapters/schema.sql +++ b/dbt/include/fabric/macros/adapters/schema.sql @@ -27,7 +27,7 @@ identifier=row[1], type=row[3] ) -%} - {% do drop_relation(schema_relation) %} + {% do drop_relation_if_exists(schema_relation) %} {%- endfor %} {% call statement('drop_schema') -%} diff --git a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql index bac562c..49547ff 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql @@ -3,8 +3,14 @@ {%- set full_refresh_mode = (should_full_refresh()) -%} {% set target_relation = this.incorporate(type='table') %} - {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%} + {%- set relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%} + {%- set existing_relation = none %} + {% if (relation.type == target_relation.type) and (relation.identifier == target_relation.identifier) and (relation.schema == target_relation.schema) and (relation.database == target_relation.database) %} + {% set existing_relation = target_relation %} + {% elif (relation.type != target_relation.type) and (relation.identifier == target_relation.identifier) and (relation.schema == target_relation.schema) and (relation.database == target_relation.database) %} + {% set existing_relation = get_or_create_relation(relation.database, relation.schema, relation.identifier, relation.type)[1] %} + {% endif %} {# {%- set relations_list = fabric__get_relation_without_caching(target_relation) -%} {%- set existing_relation = none %} {% if (relations_list|length == 1) and (relations_list[0][2] == target_relation.schema) @@ -19,6 +25,9 @@ {{ log("existing relation : "~existing_relation ~ " type "~ existing_relation.type ~ " is view? "~existing_relation.is_view) }} {{ log("target relation: " ~target_relation ~ " type "~ target_relation.type ~ " is view? "~target_relation.is_view) }} #} + {{ log("existing relation : "~existing_relation ~ " type "~ existing_relation.type ~ " is view? "~existing_relation.is_view) }} + {{ log("target relation: " ~target_relation ~ " type "~ target_relation.type ~ " is view? "~target_relation.is_view) }} + -- configs {%- set unique_key = config.get('unique_key') -%} {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} @@ -30,28 +39,25 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} {% if existing_relation is none %} - {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} {%- endcall -%} {% elif existing_relation.is_view %} - {#-- Can't overwrite a view with a table - we must drop --#} {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} - {{ drop_relation(existing_relation) }} + {{ drop_relation_if_exists(existing_relation) }} + {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} {%- endcall -%} {% elif full_refresh_mode %} - {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} {%- endcall -%} {% else %} - {%- call statement('create_tmp_relation') -%} {{ get_create_table_as_sql(True, temp_relation, sql)}} {%- endcall -%} @@ -73,7 +79,7 @@ {%- endcall -%} {% endif %} - {% do drop_relation(temp_relation) %} + {% do drop_relation_if_exists(temp_relation) %} {{ run_hooks(post_hooks, inside_transaction=True) }} {% set target_relation = target_relation.incorporate(type='table') %} diff --git a/dbt/include/fabric/macros/materializations/models/table/clone.sql b/dbt/include/fabric/macros/materializations/models/table/clone.sql index 292c882..b2e9d24 100644 --- a/dbt/include/fabric/macros/materializations/models/table/clone.sql +++ b/dbt/include/fabric/macros/materializations/models/table/clone.sql @@ -24,7 +24,7 @@ {%- set target_relation = this.incorporate(type='table') -%} {% call statement('main') %} - {{ drop_relation(target_relation) }} + {{ drop_relation_if_exists(target_relation) }} {{ create_or_replace_clone(target_relation, defer_relation) }} {% endcall %} {{ return({'relations': [target_relation]}) }} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index f9ee7fb..2977dc3 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -13,7 +13,7 @@ {% if (existing_relation != none) %} -- drop the temp relations if they exist already in the database - {{ drop_relation(backup_relation) }} + {{ drop_relation_if_exists(backup_relation) }} -- Rename target relation as backup relation {{ adapter.rename_relation(existing_relation, backup_relation) }} {% endif %} @@ -41,7 +41,7 @@ -- finally, drop the foreign key references if exists {{ drop_fk_indexes_on_table(backup_relation) }} -- drop existing/backup relation after the commit - {{ drop_relation(backup_relation) }} + {{ drop_relation_if_exists(backup_relation) }} {% endif %} -- Add constraints including FK relation. {{ fabric__build_model_constraints(target_relation) }} diff --git a/dbt/include/fabric/macros/materializations/models/view/view.sql b/dbt/include/fabric/macros/materializations/models/view/view.sql index 1c9041c..40f7752 100644 --- a/dbt/include/fabric/macros/materializations/models/view/view.sql +++ b/dbt/include/fabric/macros/materializations/models/view/view.sql @@ -12,7 +12,7 @@ {% if (existing_relation != none) %} -- drop the temp relations if they exist already in the database - {{ drop_relation(backup_relation) }} + {{ drop_relation_if_exists(backup_relation) }} -- Rename target relation as backup relation {{ adapter.rename_relation(existing_relation, backup_relation) }} {% endif %} @@ -35,7 +35,7 @@ {{ run_hooks(post_hooks, inside_transaction=True) }} {{ adapter.commit() }} {% if (backup_relation != none) %} - {{ drop_relation(backup_relation) }} + {{ drop_relation_if_exists(backup_relation) }} {% endif %} {{ run_hooks(post_hooks, inside_transaction=False) }} {{ return({'relations': [target_relation]}) }} diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index 8a5b0bd..a37ae64 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -1,6 +1,6 @@ {% macro fabric__post_snapshot(staging_relation) %} -- Clean up the snapshot temp table - {% do drop_relation(staging_relation) %} + {% do drop_relation_if_exists(staging_relation) %} {% endmacro %} --Due to Alter not being supported, have to rely on this for temporarily diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py index 071e4e1..186ae69 100644 --- a/tests/functional/adapter/test_caching.py +++ b/tests/functional/adapter/test_caching.py @@ -1,3 +1,4 @@ +import pytest from dbt.tests.adapter.caching.test_caching import ( BaseCachingLowercaseModel, BaseCachingSelectedSchemaOnly, @@ -10,6 +11,7 @@ class TestCachingLowerCaseModel(BaseCachingLowercaseModel): pass +@pytest.mark.skip(reason="Fabric DW does not support Case Insensivity.") class TestCachingUppercaseModel(BaseCachingUppercaseModel): pass