Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updating dispatching methods to ensure dbt-synapse adapter can use ad… #178

Merged
merged 17 commits into from
May 30, 2024
Merged

Updating dispatching methods to ensure dbt-synapse adapter can use ad… #178

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e886a69
Updating dispatching methods to ensure dbt-synapse adapter can use ad…
prdpsvs May 21, 2024
eb7e3f8
additional changes
prdpsvs May 21, 2024
2d34916
Updating dbt-fabric adapter to stay compatible with dbt-synapse adapter
prdpsvs May 22, 2024
36187bf
changes to drop_relation_if_exists
prdpsvs May 23, 2024
35110fe
Updated tests and drop_relation
prdpsvs May 23, 2024
8cc9227
adding temp relation to table materialization to support temp view dr…
prdpsvs May 27, 2024
1e72de2
adding a log
prdpsvs May 27, 2024
4aa5807
dropping and creating temp relation in table materialization and remo…
prdpsvs May 27, 2024
916d48d
removing hardcoded fabric__ references to not to break dbt-synapse an…
prdpsvs May 27, 2024
7bde13e
moving temp relation drop to table, incremental and snapshot material…
prdpsvs May 27, 2024
180ecc0
adding drop relation to create_table_as to support test_store_tests
prdpsvs May 27, 2024
362e415
adding log statements
prdpsvs May 28, 2024
a13807a
removing comments
prdpsvs May 28, 2024
abb9131
Update
prdpsvs May 28, 2024
59e81b8
Update dbt/include/fabric/macros/adapters/columns.sql
prdpsvs May 30, 2024
f172f8c
Update dbt/include/fabric/macros/materializations/models/incremental/…
prdpsvs May 30, 2024
0f8e3cb
Resolving comments
prdpsvs May 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dbt/adapters/fabric/__version__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version = "1.8.5"
version = "1.8.6"
16 changes: 14 additions & 2 deletions dbt/include/fabric/macros/adapters/metadata.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@
information_schema
{%- endmacro %}

{% macro get_use_database_sql(database) %}
{{ return(adapter.dispatch('get_use_database_sql', 'dbt')(database)) }}
{% endmacro %}

{%- macro default__get_use_database_sql(database) -%}
prdpsvs marked this conversation as resolved.
Show resolved Hide resolved
{%- endmacro -%}


{%- macro fabric__get_use_database_sql(database) -%}
USE [{{database}}];
prdpsvs marked this conversation as resolved.
Show resolved Hide resolved
{%- endmacro -%}

{% macro fabric__list_schemas(database) %}
{% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
select name as [schema]
Expand All @@ -27,7 +39,7 @@

{% macro fabric__list_relations_without_caching(schema_relation) -%}
{% call statement('list_relations_without_caching', fetch_result=True) -%}
USE [{{ schema_relation.database }}];
{{ get_use_database_sql(schema_relation.database) }}
with base as (
select
DB_NAME() as [database],
Expand All @@ -51,7 +63,7 @@

{% macro fabric__get_relation_without_caching(schema_relation) -%}
{% call statement('get_relation_without_caching', fetch_result=True) -%}
USE [{{ schema_relation.database }}];
{{ get_use_database_sql(schema_relation.database) }}
with base as (
select
DB_NAME() as [database],
Expand Down
6 changes: 3 additions & 3 deletions dbt/include/fabric/macros/adapters/relation.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

{% if relation.type == 'view' -%}
{% call statement('find_references', fetch_result=true) %}
USE [{{ relation.database }}];
{{ get_use_database_sql(relation.database) }}
select
sch.name as schema_name,
obj.name as view_name
Expand Down Expand Up @@ -44,13 +44,13 @@
{%- else -%}
{{ exceptions.raise_not_implemented('Invalid relation being dropped: ' ~ relation) }}
{% endif %}
USE [{{ relation.database }}];
{{ get_use_database_sql(relation.database) }}
EXEC('DROP {{ relation.type }} IF EXISTS {{ relation.include(database=False) }};');
{% endmacro %}

{% macro fabric__rename_relation(from_relation, to_relation) -%}
{% call statement('rename_relation') -%}
USE [{{ from_relation.database }}];
{{ get_use_database_sql(from_relation.database) }}
EXEC sp_rename '{{ from_relation.schema }}.{{ from_relation.identifier }}', '{{ to_relation.identifier }}'
{%- endcall %}
{% endmacro %}
Expand Down
2 changes: 1 addition & 1 deletion dbt/include/fabric/macros/adapters/schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

{% macro fabric__create_schema_with_authorization(relation, schema_authorization) -%}
{% call statement('create_schema') -%}
USE [{{ relation.database }}];
{{ get_use_database_sql(relation.database) }}
IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
BEGIN
EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]')
Expand Down
21 changes: 11 additions & 10 deletions dbt/include/fabric/macros/materializations/models/incremental/incremental.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,21 @@

{%- set full_refresh_mode = (should_full_refresh()) -%}
{% set target_relation = this.incorporate(type='table') %}
{%- set relations_list = fabric__get_relation_without_caching(target_relation) -%}
{%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%}

{# {%- set relations_list = fabric__get_relation_without_caching(target_relation) -%}
{%- set existing_relation = none %}
{% if (relations_list|length == 1) and (relations_list[0][2] == target_relation.schema)
and (relations_list[0][1] == target_relation.identifier) and (relations_list[0][3] == target_relation.type)%}
{% set existing_relation = target_relation %}
{% elif (relations_list|length == 1) and (relations_list[0][2] == target_relation.schema)
and (relations_list[0][1] == target_relation.identifier) and (relations_list[0][3] != target_relation.type) %}
{% set existing_relation = get_or_create_relation(relations_list[0][0], relations_list[0][2] , relations_list[0][1] , relations_list[0][3])[1] %}
{% endif %}
{% endif %} #}

{{ log("Full refresh mode" ~ full_refresh_mode)}}
{# {{ log("Full refresh mode" ~ full_refresh_mode)}}
{{ log("existing relation : "~existing_relation ~ " type "~ existing_relation.type ~ " is view? "~existing_relation.is_view) }}
{{ log("target relation: " ~target_relation ~ " type "~ target_relation.type ~ " is view? "~target_relation.is_view) }}
{{ log("target relation: " ~target_relation ~ " type "~ target_relation.type ~ " is view? "~target_relation.is_view) }} #}

-- configs
{%- set unique_key = config.get('unique_key') -%}
Expand All @@ -31,28 +32,28 @@
{% if existing_relation is none %}

{%- call statement('main') -%}
{{ fabric__create_table_as(False, target_relation, sql)}}
{{ get_create_table_as_sql(False, target_relation, sql)}}
{%- endcall -%}

{% elif existing_relation.is_view %}

{#-- Can't overwrite a view with a table - we must drop --#}
{{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }}
{{ drop_relation_if_exists(existing_relation) }}
{{ drop_relation(existing_relation) }}
{%- call statement('main') -%}
{{ fabric__create_table_as(False, target_relation, sql)}}
{{ get_create_table_as_sql(False, target_relation, sql)}}
{%- endcall -%}

{% elif full_refresh_mode %}

{%- call statement('main') -%}
{{ fabric__create_table_as(False, target_relation, sql)}}
{{ get_create_table_as_sql(False, target_relation, sql)}}
{%- endcall -%}

{% else %}

{%- call statement('create_tmp_relation') -%}
{{ fabric__create_table_as(True, temp_relation, sql)}}
{{ get_create_table_as_sql(True, temp_relation, sql)}}
{%- endcall -%}
{% do adapter.expand_target_column_types(
from_relation=temp_relation,
Expand All @@ -72,7 +73,7 @@
{%- endcall -%}
{% endif %}

{% do drop_relation_if_exists(temp_relation) %}
{% do drop_relation(temp_relation) %}
{{ run_hooks(post_hooks, inside_transaction=True) }}

{% set target_relation = target_relation.incorporate(type='table') %}
Expand Down
2 changes: 1 addition & 1 deletion dbt/include/fabric/macros/materializations/models/table/clone.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
{%- set target_relation = this.incorporate(type='table') -%}

{% call statement('main') %}
{{ fabric__drop_relation_script(target_relation) }}
{{ drop_relation(target_relation) }}
{{ create_or_replace_clone(target_relation, defer_relation) }}
{% endcall %}
{{ return({'relations': [target_relation]}) }}
Expand Down
6 changes: 3 additions & 3 deletions dbt/include/fabric/macros/materializations/models/table/create_table_as.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
{% set tmp_relation = relation.incorporate(
path={"identifier": relation.identifier.replace("#", "") ~ '_temp_view'},
type='view')-%}
{% do run_query(fabric__drop_relation_script(tmp_relation)) %}
{% do run_query(drop_relation(tmp_relation)) %}

{% set contract_config = config.get('contract') %}

{{ fabric__create_view_as(tmp_relation, sql) }}
{{ get_create_view_as_sql(tmp_relation, sql) }}
{% if contract_config.enforced %}

CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]
Expand All @@ -27,6 +27,6 @@
EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);');
{% endif %}

{{ fabric__drop_relation_script(tmp_relation) }}
{{ drop_relation(tmp_relation) }}

{% endmacro %}
13 changes: 3 additions & 10 deletions dbt/include/fabric/macros/materializations/models/table/table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,9 @@

-- Load target relation
{%- set target_relation = this.incorporate(type='table') %}
-- Load existing relation
{%- set relation = fabric__get_relation_without_caching(this) %}

{% set existing_relation = none %}
{% if (relation|length == 1) %}
{% set existing_relation = get_or_create_relation(relation[0][0], relation[0][2] , relation[0][1] , relation[0][3])[1] %}
{% endif %}
{%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%}

{%- set backup_relation = none %}
{{log("Existing Relation type is "~ existing_relation.type)}}
{% if (existing_relation != none and existing_relation.type == "table") %}
{%- set backup_relation = make_backup_relation(target_relation, 'table') -%}
{% elif (existing_relation != none and existing_relation.type == "view") %}
Expand All @@ -20,7 +13,7 @@

{% if (existing_relation != none) %}
-- drop the temp relations if they exist already in the database
{{ drop_relation_if_exists(backup_relation) }}
{{ drop_relation(backup_relation) }}
-- Rename target relation as backup relation
{{ adapter.rename_relation(existing_relation, backup_relation) }}
{% endif %}
Expand Down Expand Up @@ -48,7 +41,7 @@
-- finally, drop the foreign key references if exists
{{ drop_fk_indexes_on_table(backup_relation) }}
-- drop existing/backup relation after the commit
{{ drop_relation_if_exists(backup_relation) }}
{{ drop_relation(backup_relation) }}
{% endif %}
-- Add constraints including FK relation.
{{ fabric__build_model_constraints(target_relation) }}
Expand Down
2 changes: 1 addition & 1 deletion dbt/include/fabric/macros/materializations/models/view/create_view_as.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

{%- set temp_view_sql = sql.replace("'", "''") -%}

USE [{{ relation.database }}];
{{ get_use_database_sql(relation.database) }}
{% set contract_config = config.get('contract') %}
{% if contract_config.enforced %}
{{ get_assert_columns_equivalent(sql) }}
Expand Down
14 changes: 3 additions & 11 deletions dbt/include/fabric/macros/materializations/models/view/view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
{% materialization view, adapter='fabric' -%}

{%- set target_relation = this.incorporate(type='view') -%}
{{log("Target Relation "~target_relation)}}

{%- set relation = fabric__get_relation_without_caching(this) %}
{% set existing_relation = none %}
{% if (relation|length == 1) %}
{% set existing_relation = get_or_create_relation(relation[0][0], relation[0][2] , relation[0][1] , relation[0][3])[1] %}
{% endif %}
{{log("Existing Relation "~existing_relation)}}
{%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%}

{%- set backup_relation = none %}
{{log("Existing Relation type is "~ existing_relation.type)}}
{% if (existing_relation != none and existing_relation.type == "table") %}
{%- set backup_relation = make_backup_relation(target_relation, 'table') -%}
{% elif (existing_relation != none and existing_relation.type == "view") %}
Expand All @@ -20,7 +12,7 @@

{% if (existing_relation != none) %}
-- drop the temp relations if they exist already in the database
{{ drop_relation_if_exists(backup_relation) }}
{{ drop_relation(backup_relation) }}
prdpsvs marked this conversation as resolved.
Show resolved Hide resolved
-- Rename target relation as backup relation
{{ adapter.rename_relation(existing_relation, backup_relation) }}
{% endif %}
Expand All @@ -43,7 +35,7 @@
{{ run_hooks(post_hooks, inside_transaction=True) }}
{{ adapter.commit() }}
{% if (backup_relation != none) %}
{{ drop_relation_if_exists(backup_relation) }}
{{ drop_relation(backup_relation) }}
{% endif %}
{{ run_hooks(post_hooks, inside_transaction=False) }}
{{ return({'relations': [target_relation]}) }}
Expand Down
4 changes: 2 additions & 2 deletions dbt/include/fabric/macros/materializations/snapshots/helpers.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,9 @@

{% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %}
{% set temp_relation = make_temp_relation(target_relation) %}
{% set select = fabric__snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %}
{% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %}
{% call statement('build_snapshot_staging_relation') %}
{{ create_table_as(True, temp_relation, select) }}
{{ get_create_table_as_sql(True, temp_relation, select) }}
{% endcall %}

{% do return(temp_relation) %}
Expand Down
8 changes: 2 additions & 6 deletions dbt/include/fabric/macros/materializations/snapshots/snapshot.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
{% if not target_relation_exists %}

{% set build_sql = build_snapshot_table(strategy, temp_snapshot_relation) %}
{% set final_sql = create_table_as(False, target_relation, build_sql) %}
{% set final_sql = get_create_table_as_sql(False, target_relation, build_sql) %}

{% else %}

Expand Down Expand Up @@ -75,15 +75,13 @@
insert_cols = quoted_source_columns
)
%}

{% endif %}

{% call statement('main') %}
{{ final_sql }}
{% endcall %}

fabric__drop_relation_script(temp_snapshot_relation)

{{ drop_relation_if_exists(temp_snapshot_relation) }}
{% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}

Expand All @@ -94,15 +92,13 @@
{% endif %}

{{ run_hooks(post_hooks, inside_transaction=True) }}

{{ adapter.commit() }}

{% if staging_table is defined %}
{% do post_snapshot(staging_table) %}
{% endif %}

{{ run_hooks(post_hooks, inside_transaction=False) }}

{{ return({'relations': [target_relation]}) }}

{% endmaterialization %}
Loading
Loading