Release 0.7.1

Datavault-UK · Dec 18, 2020 · 7ce6cd3 · 7ce6cd3
1 parent a90f480
commit 7ce6cd3
Show file tree

Hide file tree

Showing 25 changed files with 271 additions and 126 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,14 +1,16 @@
+## Contributing new features
+
+Please refer to our contribution guidelines over on our [development repository](https://github.com/Datavault-UK/dbtvault-dev/blob/master/CONTRIBUTING.md)
+
 ## We'd love to hear from you
 
 dbtvault is very much a work in progress – we’re constantly adding quality of life improvements and will be adding
 new table types regularly.
 
-We know that it deserves new features, that the code base can be tidied up and the SQL better tuned. 
-
-Rest assured we’re working on it for future releases – [our roadmap contains information on what’s coming](roadmap.md).
+Rest assured we’re working on future releases – [our roadmap contains information on what’s coming](https://dbtvault.readthedocs.io/en/latest/roadmap/).
 
 If you spot anything you’d like to bring to our attention, have a request for new features, have spotted an improvement we could make, 
-or want to tell us about a typo or bug, then please don’t hesitate to let us know via [Github](https://github.com/Datavault-UK/dbtvault/issues). 
+or want to tell us about a typo or bug, then please don’t hesitate to let us know via [github](https://github.com/Datavault-UK/dbtvault/issues). 
 
 We’d rather know you are making active use of this package than hearing nothing from all of you out there! 
 
@@ -30,8 +32,4 @@ We'd love to add new features to make this package even more useful for the comm
 please feel free to submit ideas and thoughts!
 
 ### If it's an idea, feedback or a general inquiry
-Create a post with as much detail as possible; We'll be happy to reply and work with you.
-
-## Pull requests
-If you've developed something which we can add via a pull request, we're more than happy to consider it, but we'd
-like to discuss the changes first.
+Create a post with as much detail as possible; We'll be happy to reply and work with you.
diff --git a/macros/internal/alias.sql b/macros/internal/alias.sql
@@ -1,6 +1,6 @@
 {%- macro alias(alias_config=none, prefix=none) -%}
 
-    {{- adapter.dispatch('alias', packages = ['dbtvault'])(alias_config=alias_config, prefix=prefix) -}}
+    {{- adapter.dispatch('alias', packages = var('adapter_packages', ['dbtvault']))(alias_config=alias_config, prefix=prefix) -}}
 
 {%- endmacro %}
 

diff --git a/macros/internal/alias_all.sql b/macros/internal/alias_all.sql
@@ -1,6 +1,6 @@
 {%- macro alias_all(columns=none, prefix=none) -%}
 
-    {{- adapter.dispatch('alias_all', packages = ['dbtvault'])(columns=columns, prefix=prefix) -}}
+    {{- adapter.dispatch('alias_all', packages = var('adapter_packages', ['dbtvault']))(columns=columns, prefix=prefix) -}}
 
 {%- endmacro %}
 

diff --git a/macros/internal/as_constant.sql b/macros/internal/as_constant.sql
@@ -1,6 +1,6 @@
 {%- macro as_constant(column_str=none) -%}
 
-    {{- adapter.dispatch('as_constant', packages = ['dbtvault'])(column_str=column_str) -}}
+    {{- adapter.dispatch('as_constant', packages = var('adapter_packages', ['dbtvault']))(column_str=column_str) -}}
 
 {%- endmacro %}
 

diff --git a/macros/internal/expand_column_list.sql b/macros/internal/expand_column_list.sql
@@ -14,20 +14,20 @@
 
         {%- if col is string -%}
 
-            {%- set _ = col_list.append(col) -%}
+            {%- do col_list.append(col) -%}
 
         {#- If list of lists -#}
         {%- elif col is iterable and col is not string -%}
 
             {%- if col is mapping -%}
 
-                {%- set _ = col_list.append(col) -%}
+                {%- do col_list.append(col) -%}
 
             {%- else -%}
 
                 {%- for cols in col -%}
 
-                    {%- set _ = col_list.append(cols) -%}
+                    {%- do col_list.append(cols) -%}
 
                 {%- endfor -%}
 

diff --git a/macros/internal/multikey.sql b/macros/internal/multikey.sql
@@ -1,6 +1,6 @@
 {%- macro multikey(columns, prefix=none, condition=none, operator='AND') -%}
 
-    {{- adapter.dispatch('multikey', packages = ['dbtvault'])(columns=columns, prefix=prefix, condition=condition, operator=operator) -}}
+    {{- adapter.dispatch('multikey', packages = var('adapter_packages', ['dbtvault']))(columns=columns, prefix=prefix, condition=condition, operator=operator) -}}
 
 {%- endmacro %}
 

diff --git a/macros/internal/process_macros.sql b/macros/internal/process_macros.sql
@@ -0,0 +1,96 @@
+{%- macro process_excludes(source_relation=none, derived_columns=none, columns=none) -%}
+
+{%- set exclude_columns_list = [] -%}
+{%- set include_columns = [] -%}
+{%- if exclude_columns is none -%}
+    {%- set exclude_columns = false -%}
+{% endif %}
+
+{#- getting all the source columns -#}
+
+{%- set source_columns = dbtvault.source_columns(source_relation=source_relation) -%}
+
+{%- if columns is mapping -%}
+
+    {%- for col in columns -%}
+
+        {# Checks if the exclude flag is present and then creates a exclude list to pass to NEED BETTER NAME FOR MACRO #}
+        {%- if columns[col] is mapping and columns[col].exclude_columns -%}
+
+            {%- for flagged_cols in columns[col]['columns'] -%}
+
+                {%- do exclude_columns_list.append(flagged_cols) -%}
+
+            {%- endfor -%}
+
+            {%- set include_columns = dbtvault.process_include_columns(primary_set_list=derived_columns, secondary_set_list=source_columns, exclude_columns_list=exclude_columns_list) -%}
+
+            {#- Updates the the apropriate hashdiff to contain the columns we do want to hash  -#}
+            {%- do columns[col].update({'columns': include_columns}) -%}
+            {%- do columns[col].pop('exclude_columns') -%}
+            {%- set include_columns = [] -%}
+            {%- set exclude_columns = [] -%}
+
+        {%- endif -%}
+    {%- endfor -%}
+{%- endif -%}
+
+{%- do return(columns) -%}
+
+
+{%- endmacro -%}
+
+
+{%- macro process_include_columns(primary_set_list=none, secondary_set_list=none, exclude_columns_list=none) -%}
+
+{%- set include_columns = [] -%}
+
+{%- if exclude_columns is none -%}
+    {%- set exclude_columns_list = [] -%}
+{%- endif -%}
+
+{# Appending primary list items not in exclude columns #}
+{%- if primary_set_list is not none -%}
+
+    {%- for primary_col in primary_set_list -%}
+
+        {%- if primary_col not in exclude_columns_list -%}
+
+            {%- if primary_set_list is mapping -%}
+                {%- set primary_str = dbtvault.as_constant(primary_col) -%}
+                {%- do include_columns.append(primary_str) -%}
+                {%- do exclude_columns_list.append(primary_str)  -%}
+            {%- else -%}
+                {%- do include_columns.append(primary_col) -%}
+                {%- do exclude_columns_list.append(primary_col) -%}
+            {%- endif -%}
+
+        {%- endif -%}
+
+    {%- endfor -%}
+
+{%- endif -%}
+
+{# Apending the secondary list items not in the priamry list or the exclude list #}
+{%- if secondary_set_list is not none -%}
+
+    {%- for secondary_col in secondary_set_list -%}
+
+        {%- if secondary_col not in exclude_columns_list -%}
+
+            {%- if secondary_set_list is mapping -%}
+                {%- set secondary_str = dbtvault.as_constant(secondary_col) -%}
+                {%- do include_columns.append(secondary_str) -%}
+            {%- else -%}
+                {%- do include_columns.append(secondary_col) -%}
+            {%- endif -%}
+
+        {%- endif -%}
+
+    {% endfor -%}
+
+{%- endif -%}
+
+{%- do return(include_columns) -%}
+
+{%- endmacro -%}
diff --git a/macros/materialisations/helpers.sql b/macros/materialisations/helpers.sql
@@ -7,16 +7,16 @@
 {%- macro replace_placeholder_with_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) -%}
 
     {% set macro = adapter.dispatch('replace_placeholder_with_filter',
-                                    packages = ['dbtvault'])(core_sql=core_sql,
-                                                             timestamp_field=timestamp_field,
-                                                             start_timestamp=start_timestamp,
-                                                             stop_timestamp=stop_timestamp,
-                                                             offset=offset,
-                                                             period=period) %}
+                                    packages = var('adapter_packages', ['dbtvault']))(core_sql=core_sql,
+                                                                                      timestamp_field=timestamp_field,
+                                                                                      start_timestamp=start_timestamp,
+                                                                                      stop_timestamp=stop_timestamp,
+                                                                                      offset=offset,
+                                                                                      period=period) %}
     {% do return(macro) %}
 {%- endmacro %}
 
-{% macro snowflake__replace_placeholder_with_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %}
+{% macro default__replace_placeholder_with_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %}
 
     {%- set period_filter -%}
             (TO_DATE({{ timestamp_field }}) >= DATE_TRUNC('{{ period }}', TO_DATE('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}') AND
@@ -35,7 +35,7 @@
 {%- macro get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
 
     {% set macro = adapter.dispatch('get_period_filter_sql',
-                                    packages = ['dbtvault'])(target_cols_csv=target_cols_csv,
+                                    packages = var('adapter_packages', ['dbtvault']))(target_cols_csv=target_cols_csv,
                                                              base_sql=base_sql,
                                                              timestamp_field=timestamp_field,
                                                              period=period,
@@ -45,7 +45,7 @@
     {% do return(macro) %}
 {%- endmacro %}
 
-{% macro snowflake__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
+{% macro default__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
 
     {%- set filtered_sql = {'sql': base_sql} -%}
 
@@ -63,7 +63,7 @@
 {%- macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
 
     {% set macro = adapter.dispatch('get_period_boundaries',
-                                    packages = ['dbtvault'])(target_schema=target_schema,
+                                    packages = var('adapter_packages', ['dbtvault']))(target_schema=target_schema,
                                                              target_table=target_table,
                                                              timestamp_field=timestamp_field,
                                                              start_date=start_date,
@@ -73,7 +73,7 @@
     {% do return(macro) %}
 {%- endmacro %}
 
-{% macro snowflake__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
+{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
 
     {% set period_boundary_sql -%}
         with data as (
@@ -107,14 +107,14 @@
 {%- macro get_period_of_load(period, offset, start_timestamp) -%}
 
     {% set macro = adapter.dispatch('get_period_of_load',
-                                    packages = ['dbtvault'])(period=period,
+                                    packages = var('adapter_packages', ['dbtvault']))(period=period,
                                                              offset=offset,
                                                              start_timestamp=start_timestamp) %}
 
     {% do return(macro) %}
 {%- endmacro %}
 
-{%- macro snowflake__get_period_of_load(period, offset, start_timestamp) -%}
+{%- macro default__get_period_of_load(period, offset, start_timestamp) -%}
 
     {% set period_of_load_sql -%}
         SELECT DATE_TRUNC('{{ period }}', DATEADD({{ period }}, {{ offset }}, TO_DATE('{{start_timestamp}}'))) AS period_of_load

diff --git a/macros/materialisations/helpers_snowflake_schema.yml b/macros/materialisations/helpers_snowflake_schema.yml
@@ -1,7 +1,7 @@
 version: 2
 
 macros:
-  - name: snowflake__replace_placeholder_with_filter
+  - name: default__replace_placeholder_with_filter
     description: |
                  {{ doc("macro__replace_placeholder_with_filter") }}
 
@@ -26,7 +26,7 @@ macros:
         type: string
         description: '{{ doc("arg__period_materialisation__period") }}'
 
-  - name: snowflake__get_period_filter_sql
+  - name: default__get_period_filter_sql
     description: |
                  {{ doc("macro__get_period_filter_sql") }}
 
@@ -54,7 +54,7 @@ macros:
         type: string
         description: '{{ doc("arg__period_materialisation__offset") }}'
 
-  - name: snowflake__get_period_boundaries
+  - name: default__get_period_boundaries
     description: |
                  {{ doc("macro__get_period_boundaries") }}
 
@@ -79,7 +79,7 @@ macros:
         type: string
         description: '{{ doc("arg__period_materialisation__period") }}'
 
-  - name: snowflake__get_period_of_load
+  - name: default__get_period_of_load
     description: |
                  {{ doc("macro__get_period_of_load") }}
 

diff --git a/macros/materialisations/vault_insert_by_period_materialization.sql b/macros/materialisations/vault_insert_by_period_materialization.sql
@@ -29,6 +29,8 @@
                                                                        0, period) %}
         {% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
 
+        {% do to_drop.append(tmp_relation) %}
+
     {% elif existing_relation.is_view or full_refresh_mode %}
         {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
         {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
@@ -43,6 +45,7 @@
                                                                        0, period) %}
         {% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
 
+        {% do to_drop.append(tmp_relation) %}
         {% do to_drop.append(backup_relation) %}
     {% else %}
 
@@ -88,13 +91,14 @@
             {%- set rows_inserted = (load_result(insert_query_name)['status'].split(" "))[1] | int -%}
 
             {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%}
-            {%- set _ = loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %}
+            {%- do loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %}
 
             {{ dbt_utils.log_info("Ran for {} {} of {} ({}); {} records inserted [{}]".format(period, iteration_number,
                                                                                               period_boundaries.num_periods,
                                                                                               period_of_load, rows_inserted,
                                                                                               model.unique_id)) }}
 
+            {% do to_drop.append(tmp_relation) %}
             {% do adapter.commit() %}
 
         {% endfor %}
@@ -123,7 +127,9 @@
     {{ run_hooks(post_hooks, inside_transaction=True) }}
 
     {% for rel in to_drop %}
-        {{ drop_relation_if_exists(backup_relation) }}
+        {% if rel.type is not none %}
+            {% do adapter.drop_relation(rel) %}
+        {% endif %}
     {% endfor %}
 
     {{ run_hooks(post_hooks, inside_transaction=False) }}