Release 0.9.7

Datavault-UK · Jul 19, 2023 · 82c61bf · 82c61bf
1 parent 3db7cc2
commit 82c61bf
Show file tree

Hide file tree

Showing 66 changed files with 998 additions and 714 deletions.
diff --git a/README.md b/README.md
@@ -89,4 +89,4 @@ Talk to our developers and other members of our growing community, get support a
 [View our contribution guidelines](CONTRIBUTING.md)
 
 ## License
-[Apache 2.0](LICENSE.md)
+[Apache 2.0](LICENSE)
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: automate_dv
-version: 0.9.5
+version: 0.9.7
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 config-version: 2
 

diff --git a/macros/internal/helpers/dateadd.sql b/macros/internal/helpers/dateadd.sql
@@ -18,8 +18,8 @@
 {% macro sqlserver__dateadd(datepart, interval, from_date_or_timestamp) %}
 
     dateadd(
-        millisecond,
-        86399999,
+        {{ datepart }},
+        {{ interval }},
         CAST({{ from_date_or_timestamp }} AS DATETIME2)
     )
 

diff --git a/macros/internal/helpers/stage_processing_macros/extract_column_names.sql b/macros/internal/helpers/stage_processing_macros/extract_column_names.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro extract_column_names(columns_dict=none) -%}
 
     {%- set extracted_column_names = [] -%}

diff --git a/macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql b/macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro extract_null_column_names(columns_dict=none) -%}
 
     {%- set extracted_column_names = [] -%}

diff --git a/macros/internal/helpers/stage_processing_macros/print_list.sql b/macros/internal/helpers/stage_processing_macros/print_list.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro print_list(list_to_print=none, indent=4, columns_to_escape=none) -%}
 
     {%- for col_name in list_to_print -%}

diff --git a/macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql b/macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro process_columns_to_escape(derived_columns_list=none) -%}
 
     {%- if derived_columns_list -%}

diff --git a/macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql b/macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro process_columns_to_select(columns_list=none, exclude_columns_list=none) -%}
 
     {% set columns_list = columns_list | list %}

diff --git a/macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql b/macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql
@@ -3,8 +3,6 @@
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
-AutomateDV (f.k.a automate_dv)
-
 {%- macro process_hash_column_excludes(hash_columns=none, source_columns=none) -%}
 
     {%- set processed_hash_columns = {} -%}

diff --git a/macros/internal/helpers/timestamp_add.sql b/macros/internal/helpers/timestamp_add.sql
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) Business Thinking Ltd. 2019-2023
+ * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
+ */
+
+{% macro timestamp_add(datepart, interval, from_date_or_timestamp) %}
+    {{ return(adapter.dispatch('timestamp_add', 'automate_dv')(datepart=datepart, interval=interval,
+                                                               from_date_or_timestamp=from_date_or_timestamp)) }}
+{%- endmacro -%}
+
+{%- macro default__timestamp_add(datepart, interval, from_date_or_timestamp) -%}
+
+    {%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%}
+        {{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }}
+    {%- elif datepart == 'microsecond' -%}
+        {{ automate_dv.dateadd('microsecond', 1, from_date_or_timestamp) }}
+    {%- elif datepart == 'millisecond' -%}
+        {{ automate_dv.dateadd('microsecond', 999, from_date_or_timestamp) }}
+    {%- elif datepart == 'second' -%}
+        {{ automate_dv.dateadd('millisecond', 999, from_date_or_timestamp) }}
+    {%- elif datepart == 'minute' -%}
+        {{ automate_dv.dateadd('millisecond', 5999, from_date_or_timestamp) }}
+    {%- elif datepart == 'hour' -%}
+        {{ automate_dv.dateadd('millisecond', 3599999, from_date_or_timestamp) }}
+    {%- endif -%}
+
+{%- endmacro -%}
+
+{% macro bigquery__timestamp_add(datepart, interval, from_date_or_timestamp) %}
+
+{%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%}
+    {{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }}
+{%- elif datepart == 'microsecond' -%}
+    TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 1 microsecond)
+{%- elif datepart == 'millisecond' -%}
+    TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 microsecond)
+{%- elif datepart == 'second' -%}
+    TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 millisecond)
+{%- elif datepart == 'minute' -%}
+    TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 5999 millisecond)
+{%- elif datepart == 'hour' -%}
+    TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 3599999 millisecond)
+{%- endif -%}
+
+{% endmacro %}
+
+
diff --git a/macros/internal/metadata_processing/get_escape_characters.sql b/macros/internal/metadata_processing/get_escape_characters.sql
@@ -5,9 +5,30 @@
 
 {% macro get_escape_characters() -%}
 
-    {%- set escape_char_left, escape_char_right = adapter.dispatch('get_escape_characters', 'automate_dv')() -%}
+    {%- set default_escape_char_left, default_escape_char_right = adapter.dispatch('get_escape_characters', 'automate_dv')() -%}
 
-    {%- do return((var('escape_char_left', escape_char_left), var('escape_char_right', escape_char_right))) -%}
+    {%- if ((var('escape_char_left', default_escape_char_left) == '') and (var('escape_char_right', default_escape_char_right) == '')) -%}
+        {%- set warning_message = 'Invalid escape_char_left and escape_char_right value provided. ' +
+                                  'Using platform defaults ({}{})'.format(default_escape_char_left, default_escape_char_right) -%}
+        {%- set escape_chars = (default_escape_char_left, default_escape_char_right) -%}
+
+    {%- elif var('escape_char_left', default_escape_char_left) == '' -%}
+        {%- set warning_message = 'Invalid escape_char_left value provided. Using platform default ({})'.format(default_escape_char_left) -%}
+        {%- set escape_chars = (default_escape_char_left, var('escape_char_right', default_escape_char_right)) -%}
+
+    {%- elif var('escape_char_right', default_escape_char_right) == '' -%}
+        {%- set warning_message = 'Invalid escape_char_right value provided. Using platform default ({})'.format(default_escape_char_right) -%}
+        {%- set escape_chars = (var('escape_char_left', default_escape_char_left), default_escape_char_right) -%}
+
+    {%- else -%}
+       {%- set escape_chars = (var('escape_char_left', default_escape_char_left), var('escape_char_right', default_escape_char_right)) -%}
+    {%- endif -%}
+
+    {%- if execute and warning_message -%}
+        {%- do exceptions.warn(warning_message) -%}
+    {%- endif -%}
+
+    {%- do return(escape_chars) -%}
 
 {%- endmacro %}
 

diff --git a/macros/materialisations/drop_temporary.sql b/macros/materialisations/drop_temporary.sql
@@ -13,6 +13,8 @@
             DROP VIEW {{ tmp_relation }};
         {% elif target.type == 'sqlserver' %}
             DROP TABLE {{ tmp_relation }};
+        {% elif target.type == 'postgres' %}
+            DROP TABLE {{ tmp_relation }};
         {% endif %}
     {%- endcall %}
 

diff --git a/macros/materialisations/incremental_pit_bridge_replace.sql b/macros/materialisations/incremental_pit_bridge_replace.sql
@@ -4,6 +4,14 @@
  */
 
 {% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
+
+    {{ adapter.dispatch('incremental_pit_replace', 'automate_dv')(tmp_relation=tmp_relation,
+                                                                    target_relation=target_relation,
+                                                                    statement_name="main") }}
+
+{%- endmacro -%}
+
+{% macro default__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
     {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
     {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
 
@@ -17,8 +25,28 @@
 {%- endmacro %}
 
 
+{% macro databricks__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
+    {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
+
+    INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }})
+       SELECT {{ dest_cols_csv }}
+       FROM {{ tmp_relation }};
+
+{%- endmacro %}
+
+
 
 {% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
+
+    {{ adapter.dispatch('incremental_bridge_replace', 'automate_dv')(tmp_relation=tmp_relation,
+                                                                    target_relation=target_relation,
+                                                                    statement_name="main") }}
+
+{%- endmacro -%}
+
+
+{% macro default__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
     {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
     {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
 
@@ -32,4 +60,15 @@
 {%- endmacro %}
 
 
+{% macro databricks__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
+    {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
+
+    INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }})
+       SELECT {{ dest_cols_csv }}
+       FROM {{ tmp_relation }}
+    ;
+{%- endmacro %}
+
+
 
diff --git a/macros/materialisations/period_mat_helpers/check_datediff.sql b/macros/materialisations/period_mat_helpers/check_datediff.sql
@@ -5,20 +5,13 @@
 
 {%- macro check_num_periods(start_date, stop_date, period) -%}
 
-    {% set num_periods = adapter.dispatch('check_num_periods', 'automate_dv')(
-                                          start_date=start_date,
-                                          stop_date=stop_date,
-                                          period=period) %}
+    {% set num_periods = adapter.dispatch('check_num_periods',
+                                          'automate_dv')(start_date=start_date,
+                                                         stop_date=stop_date,
+                                                         period=period) %}
 
     {%- if num_periods > 100000 -%}
-        {%- set error_message -%}
-        'Max iterations is 100,000. Consider using a different datepart value (e.g. day)
-        or loading data for a shorter time period.
-        vault_insert_by materialisations are not intended for this purpose,
-        please see https://automate_dv.readthedocs.io/en/latest/materialisations/'
-        {%- endset -%}
-
-        {{- exceptions.raise_compiler_error(error_message) -}}
+        {{ automate_dv.sqlserver_max_iterations_error() }}
     {%- endif -%}
 
     {% do return(num_periods) %}
@@ -31,7 +24,7 @@
     SELECT {{ datediff('start_timestamp', 'stop_timestamp', period) }} AS NUM_PERIODS
     FROM
     (SELECT CAST('{{ start_date }}' AS {{ dbt.type_timestamp() }}) AS start_timestamp,
-        CAST(NULLIF('{{ stop_date | lower }}', 'none') AS {{ dbt.type_timestamp() }}) AS stop_timestamp)
+        CAST(NULLIF('{{ stop_date | lower }}', 'none') AS {{ dbt.type_timestamp() }}) AS stop_timestamp) AS SUBQUERY_ALIAS
     {% endset %}
     {% set num_periods_dict = automate_dv.get_query_results_as_dict(num_periods_check_sql) %}
     {% set num_periods = num_periods_dict['NUM_PERIODS'][0] | int %}

diff --git a/macros/materialisations/period_mat_helpers/error_messages.sql b/macros/materialisations/period_mat_helpers/error_messages.sql
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Business Thinking Ltd. 2019-2023
+ * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
+ */
+
+{%- macro sqlserver_datepart_too_small_error(period) -%}
+
+    {%- set error_message -%}
+    This datepart ({{ period }}) is too small and cannot be used for this purpose in MS SQL Server, consider using a different datepart value (e.g. day).
+    vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/
+    {%- endset -%}
+
+    {{- exceptions.raise_compiler_error(error_message) -}}
+
+{%- endmacro -%}
+
+{%- macro sqlserver_max_iterations_error() -%}
+
+    {%- set error_message -%}
+    Max iterations is 100,000. Consider using a different datepart value (e.g. day) or loading data for a shorter time period.
+    vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/
+    {%- endset -%}
+
+    {{- exceptions.raise_compiler_error(error_message) -}}
+
+{%- endmacro -%}