ADAP-821: Add support for replacing dynamic tables with tables/views …

…and vice versa (#753) * restructure macros directory, create separate files and directories for each ddl statement and relation type * pull create_or_replace_view into dbt-snowflake, update conditional to drop all non-views versus drop all tables * removed unnecessary create abstractions to simplify diffs * implement create or replace * test against clone replace branch * update renameable_relations and replaceable_relations to use frozensets for unmutable defaults --------- Co-authored-by: Matthew McKnight <[email protected]> Co-authored-by: colin-rogers-dbt <[email protected]>
dbt-labs · Sep 15, 2023 · eb4fd78 · eb4fd78
1 parent 01f2c3e
commit eb4fd78
Show file tree

Hide file tree

Showing 20 changed files with 218 additions and 132 deletions.
diff --git a/.changes/unreleased/Features-20230822-234732.yaml b/.changes/unreleased/Features-20230822-234732.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Support replacing tables/views with dynamic tables and vice versa
+time: 2023-08-22T23:47:32.27784-04:00
+custom:
+  Author: mikealfare
+  Issue: "753"
diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py
@@ -20,6 +20,14 @@
 class SnowflakeRelation(BaseRelation):
     type: Optional[SnowflakeRelationType] = None  # type: ignore
     quote_policy: SnowflakeQuotePolicy = field(default_factory=lambda: SnowflakeQuotePolicy())
+    renameable_relations = frozenset({SnowflakeRelationType.Table, SnowflakeRelationType.View})
+    replaceable_relations = frozenset(
+        {
+            SnowflakeRelationType.DynamicTable,
+            SnowflakeRelationType.Table,
+            SnowflakeRelationType.View,
+        }
+    )
 
     @property
     def is_dynamic_table(self) -> bool:

diff --git a/dbt/include/snowflake/macros/materializations/dynamic_table.sql b/dbt/include/snowflake/macros/materializations/dynamic_table.sql
@@ -1,68 +1,38 @@
 {% materialization dynamic_table, adapter='snowflake' %}
 
-    {% set original_query_tag = set_query_tag() %}
+    {% set query_tag = set_query_tag() %}
 
     {% set existing_relation = load_cached_relation(this) %}
     {% set target_relation = this.incorporate(type=this.DynamicTable) %}
-    {% set intermediate_relation = make_intermediate_relation(target_relation) %}
-    {% set backup_relation_type = target_relation.DynamicTable if existing_relation is none else existing_relation.type %}
-    {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}
 
-    {{ dynamic_table_setup(backup_relation, intermediate_relation, pre_hooks) }}
+    {{ run_hooks(pre_hooks) }}
 
-        {% set build_sql = dynamic_table_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
+    {% set build_sql = dynamic_table_get_build_sql(existing_relation, target_relation) %}
 
-        {% if build_sql == '' %}
-            {{ dynamic_table_execute_no_op(target_relation) }}
-        {% else %}
-            {{ dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}
-        {% endif %}
+    {% if build_sql == '' %}
+        {{ dynamic_table_execute_no_op(target_relation) }}
+    {% else %}
+        {{ dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation) }}
+    {% endif %}
 
-    {{ dynamic_table_teardown(backup_relation, intermediate_relation, post_hooks) }}
+    {{ run_hooks(post_hooks) }}
 
-    {% do unset_query_tag(original_query_tag) %}
+    {% do unset_query_tag(query_tag) %}
 
     {{ return({'relations': [target_relation]}) }}
 
 {% endmaterialization %}
 
 
-{% macro dynamic_table_setup(backup_relation, intermediate_relation, pre_hooks) %}
-
-    -- backup_relation and intermediate_relation should not already exist in the database
-    -- it's possible these exist because of a previous run that exited unexpectedly
-    {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}
-    {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}
-
-    -- drop the temp relations if they exist already in the database
-    {{ snowflake__get_drop_dynamic_table_sql(preexisting_backup_relation) }}
-    {{ snowflake__get_drop_dynamic_table_sql(preexisting_intermediate_relation) }}
-
-    {{ run_hooks(pre_hooks) }}
-
-{% endmacro %}
-
-
-{% macro dynamic_table_teardown(backup_relation, intermediate_relation, post_hooks) %}
-
-    -- drop the temp relations if they exist to leave the database clean for the next run
-    {{ snowflake__get_drop_dynamic_table_sql(backup_relation) }}
-    {{ snowflake__get_drop_dynamic_table_sql(intermediate_relation) }}
-
-    {{ run_hooks(post_hooks) }}
-
-{% endmacro %}
-
-
-{% macro dynamic_table_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
+{% macro dynamic_table_get_build_sql(existing_relation, target_relation) %}
 
     {% set full_refresh_mode = should_full_refresh() %}
 
     -- determine the scenario we're in: create, full_refresh, alter, refresh data
     {% if existing_relation is none %}
-        {% set build_sql = snowflake__get_create_dynamic_table_as_sql(target_relation, sql) %}
+        {% set build_sql = get_create_sql(target_relation, sql) %}
     {% elif full_refresh_mode or not existing_relation.is_dynamic_table %}
-        {% set build_sql = snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %}
+        {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}
     {% else %}
 
         -- get config options
@@ -74,7 +44,7 @@
             {{ exceptions.warn("No configuration changes were identified on: `" ~ target_relation ~ "`. Continuing.") }}
 
         {% elif on_configuration_change == 'apply' %}
-            {% set build_sql = snowflake__get_alter_dynamic_table_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}
+            {% set build_sql = snowflake__get_alter_dynamic_table_as_sql(existing_relation, configuration_changes, target_relation, sql) %}
         {% elif on_configuration_change == 'continue' %}
             {% set build_sql = '' %}
             {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }}
@@ -94,17 +64,17 @@
 {% endmacro %}
 
 
-{% macro dynamic_table_execute_no_op(target_relation) %}
+{% macro dynamic_table_execute_no_op(relation) %}
     {% do store_raw_result(
         name="main",
-        message="skip " ~ target_relation,
+        message="skip " ~ relation,
         code="skip",
         rows_affected="-1"
     ) %}
 {% endmacro %}
 
 
-{% macro dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}
+{% macro dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation) %}
 
     {% set grant_config = config.get('grants') %}
 

diff --git a/dbt/include/snowflake/macros/materializations/view.sql b/dbt/include/snowflake/macros/materializations/view.sql
@@ -1,7 +1,7 @@
 {% materialization view, adapter='snowflake' -%}
 
     {% set original_query_tag = set_query_tag() %}
-    {% set to_return = create_or_replace_view() %}
+    {% set to_return = snowflake__create_or_replace_view() %}
 
     {% set target_relation = this.incorporate(type='view') %}
 

diff --git a/dbt/include/snowflake/macros/relations/create.sql b/dbt/include/snowflake/macros/relations/create.sql
@@ -0,0 +1,11 @@
+{% macro snowflake__get_create_sql(relation, sql) %}
+
+    {% if relation.is_dynamic_table %}
+        {{ snowflake__get_create_dynamic_table_as_sql(relation, sql) }}
+
+    {% else %}
+        {{ default__get_create_sql(relation, sql) }}
+
+    {% endif %}
+
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/drop.sql b/dbt/include/snowflake/macros/relations/drop.sql
@@ -1,9 +1,11 @@
-{% macro snowflake__drop_relation(relation) -%}
-    {%- if relation.is_dynamic_table -%}
-        {% call statement('drop_relation', auto_begin=False) -%}
-            drop dynamic table if exists {{ relation }}
-        {%- endcall %}
-    {%- else -%}
-        {{- default__drop_relation(relation) -}}
-    {%- endif -%}
+{% macro snowflake__get_drop_sql(relation) %}
+
+    {% if relation.is_dynamic_table %}
+        {{ snowflake__get_drop_dynamic_table_sql(relation) }}
+
+    {% else %}
+        {{ default__get_drop_sql(relation) }}
+
+    {% endif %}
+
 {% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/_replace.sql
diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql b/dbt/include/snowflake/macros/relations/dynamic_table/alter.sql
@@ -1,15 +1,13 @@
 {% macro snowflake__get_alter_dynamic_table_as_sql(
-    target_relation,
-    configuration_changes,
-    sql,
     existing_relation,
-    backup_relation,
-    intermediate_relation
+    configuration_changes,
+    target_relation,
+    sql
 ) -%}
-    {{- log('Applying ALTER to: ' ~ target_relation) -}}
+    {{- log('Applying ALTER to: ' ~ existing_relation) -}}
 
     {% if configuration_changes.requires_full_refresh %}
-        {{- snowflake__get_replace_dynamic_table_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) -}}
+        {{- get_replace_sql(existing_relation, target_relation, sql) -}}
 
     {% else %}
 

diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql
@@ -1,7 +1,6 @@
 {% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%}
-    {{- log('Applying CREATE to: ' ~ relation) -}}
 
-    create or replace dynamic table {{ relation }}
+    create dynamic table {{ relation }}
         target_lag = '{{ config.get("target_lag") }}'
         warehouse = {{ config.get("snowflake_warehouse") }}
         as (

diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql
@@ -0,0 +1,12 @@
+{% macro snowflake__get_replace_dynamic_table_sql(relation, sql) %}
+
+    create or replace dynamic table {{ relation }}
+        target_lag = '{{ config.get("target_lag") }}'
+        warehouse = {{ config.get("snowflake_warehouse") }}
+        as (
+            {{ sql }}
+        )
+    ;
+    {{ snowflake__refresh_dynamic_table(relation) }}
+
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/replace.sql b/dbt/include/snowflake/macros/relations/replace.sql
@@ -0,0 +1,11 @@
+{% macro snowflake__get_replace_sql(existing_relation, target_relation, sql) %}
+
+    {% if existing_relation.is_dynamic_table and target_relation.is_dynamic_table %}
+        {{ snowflake__get_replace_dynamic_table_sql(target_relation, sql) }}
+
+    {% else %}
+        {{ default__get_replace_sql(existing_relation, target_relation, sql) }}
+
+    {% endif %}
+
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/table/drop.sql b/dbt/include/snowflake/macros/relations/table/drop.sql
@@ -0,0 +1,3 @@
+{% macro snowflake__get_drop_table_sql(relation) %}
+    drop table if exists {{ relation }} cascade
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/table/rename.sql b/dbt/include/snowflake/macros/relations/table/rename.sql
@@ -0,0 +1,3 @@
+{%- macro snowflake__get_rename_table_sql(relation, new_name) -%}
+    alter table {{ relation }} rename to {{ new_name }}
+{%- endmacro -%}
diff --git a/dbt/include/snowflake/macros/relations/table/replace.sql b/dbt/include/snowflake/macros/relations/table/replace.sql
@@ -0,0 +1,3 @@
+{% macro snowflake__get_replace_table_sql(relation, sql) %}
+    {{ snowflake__create_table_as(False, relation, sql) }}
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/view/create.sql b/dbt/include/snowflake/macros/relations/view/create.sql
@@ -28,3 +28,46 @@
 {% macro snowflake__create_view_as(relation, sql) -%}
   {{ snowflake__create_view_as_with_temp_flag(relation, sql) }}
 {% endmacro %}
+
+
+/* {#
+Vendored from dbt-core for the purpose of overwriting small pieces to support dynamics tables. This should
+eventually be retired in favor of a standardized approach. Changed line:
+
+{%- if old_relation is not none and old_relation.is_table -%}  ->
+{%- if old_relation is not none and not old_relation.is_view -%}
+#} */
+
+{% macro snowflake__create_or_replace_view() %}
+  {%- set identifier = model['alias'] -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+  {%- set target_relation = api.Relation.create(
+      identifier=identifier, schema=schema, database=database,
+      type='view') -%}
+  {% set grant_config = config.get('grants') %}
+
+  {{ run_hooks(pre_hooks) }}
+
+  -- If there's a table with the same name and we weren't told to full refresh,
+  -- that's an error. If we were told to full refresh, drop it. This behavior differs
+  -- for Snowflake and BigQuery, so multiple dispatch is used.
+  {%- if old_relation is not none and not old_relation.is_view -%}
+    {{ handle_existing_table(should_full_refresh(), old_relation) }}
+  {%- endif -%}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ get_create_view_as_sql(target_relation, sql) }}
+  {%- endcall %}
+
+  {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {{ run_hooks(post_hooks) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/view/drop.sql b/dbt/include/snowflake/macros/relations/view/drop.sql
@@ -0,0 +1,3 @@
+{% macro snowflake__get_drop_view_sql(relation) %}
+    drop view if exists {{ relation }} cascade
+{% endmacro %}
diff --git a/dbt/include/snowflake/macros/relations/view/rename.sql b/dbt/include/snowflake/macros/relations/view/rename.sql
@@ -0,0 +1,3 @@
+{%- macro snowflake__get_rename_view_sql(relation, new_name) -%}
+    alter view {{ relation }} rename to {{ new_name }}
+{%- endmacro -%}
diff --git a/dbt/include/snowflake/macros/relations/view/replace.sql b/dbt/include/snowflake/macros/relations/view/replace.sql
@@ -0,0 +1,3 @@
+{% macro snowflake__get_replace_view_sql(relation, sql) %}
+    {{ snowflake__create_view_as(relation, sql) }}
+{% endmacro %}