Release v0.9.0

dbt_project.yml

@@ -1,5 +1,5 @@
 name: dbtvault
-version: 0.8.3
+version: 0.9.0
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 config-version: 2
 
@@ -17,4 +17,5 @@ clean-targets:
 
 vars:
   hash: MD5
-  max_datetime: '{{ dbtvault.max_datetime() }}'
+  null_key_required: '-1' # Default -1, allows user to configure
+  null_key_optional: '-2' # Default -2, allows user to configure

macros/internal/helpers/is_checks.sql

@@ -16,7 +16,7 @@
 
 {%- macro is_nothing(obj) -%}
 
-    {%- if obj is none or obj is undefined or not obj -%}
+    {%- if obj is none or obj is undefined or not obj or dbtvault.is_list(obj, empty_is_false=true) -%}
         {%- do return(true) -%}
     {%- else -%}
         {%- do return(false) -%}
@@ -29,7 +29,12 @@
 {%- macro is_something(obj) -%}
 
     {%- if obj is not none and obj is defined and obj -%}
-        {%- do return(true) -%}
+        {#- if an empty list, do not consider the object something -#}
+        {% if dbtvault.is_list(empty_is_false=true) %}
+            {%- do return(true) -%}
+        {%- else -%}
+            {%- do return(false) -%}
+        {%- endif -%}
     {%- else -%}
         {%- do return(false) -%}
     {%- endif -%}
@@ -50,4 +55,4 @@
         {%- do return(false) -%}
     {%- endif -%}
 
-{%- endmacro -%}
+{%- endmacro -%}

macros/internal/helpers/logging.sql

@@ -0,0 +1,21 @@
+{% macro log_relation_sources(relation, source_count) %}
+    {{ return(adapter.dispatch('log_relation_sources', 'dbtvault')(relation=relation, source_count=source_count)) }}
+{%- endmacro -%}
+
+{% macro default__log_relation_sources(relation, source_count) %}
+
+    {%- if execute -%}
+
+        {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}.{}".format(relation.database, relation.schema, relation.identifier),
+                                                                        source_count)) -%}
+    {%- endif -%}
+{% endmacro %}
+
+{% macro databricks__log_relation_sources(relation, source_count) %}
+
+    {%- if execute -%}
+
+        {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}".format(relation.schema, relation.identifier),
+                                                                        source_count)) -%}
+    {%- endif -%}
+{% endmacro %}

macros/internal/helpers/stage_processing_macros.sql

@@ -90,3 +90,30 @@
     {%- endfor -%}
 
 {%- endmacro -%}
+
+
+{%- macro extract_null_column_names(columns_dict=none) -%}
+
+    {%- set extracted_column_names = [] -%}
+
+    {%- if columns_dict is mapping -%}
+        {%- for key, value in columns_dict.items() -%}
+            {%- if dbtvault.is_something(value) -%}
+                {% if dbtvault.is_list(value) %}
+                    {% for col_name in value %}
+                        {%- do extracted_column_names.append(col_name) -%}
+                        {%- do extracted_column_names.append(col_name ~ "_ORIGINAL") -%}
+                    {% endfor %}
+                {%  else %}
+                    {%- do extracted_column_names.append(value) -%}
+                    {%- do extracted_column_names.append(value ~ "_ORIGINAL") -%}
+                {% endif %}
+            {%- endif -%}
+        {%- endfor -%}
+
+        {%- do return(extracted_column_names) -%}
+    {%- else -%}
+        {%- do return([]) -%}
+    {%- endif -%}
+
+{%- endmacro -%}

macros/internal/metadata_processing/as_constant.sql

@@ -14,15 +14,7 @@
 
         {%- else -%}
 
-            {%- if dbtvault.is_expression(column_str) -%}
-
-                {{- return(column_str) -}}
-
-            {%- else -%}
-
-                {{- return(dbtvault.escape_column_names(column_str)) -}}
-
-            {%- endif -%}
+            {{- return(column_str) -}}
 
         {%- endif -%}
     {%- else -%}

macros/internal/metadata_processing/concat_ws.sql

@@ -6,7 +6,7 @@
 
 {%- macro default__concat_ws(string_list, separator="||") -%}
 
-    {{  "CONCAT_WS('" ~ separator ~ "', " ~ string_list | join(", ") ~ ")" }}
+    CONCAT_WS('{{ separator }}', {{ string_list | join(", ") }})
 
 {%- endmacro -%}
 
@@ -19,4 +19,10 @@
     {%- endfor -%}
     {{- '\n)' -}}
 
-{%- endmacro -%}
+{%- endmacro -%}
+
+{%- macro sqlserver__concat_ws(string_list, separator="||") -%}
+
+{{ dbtvault.default__concat_ws(string_list=string_list, separator=separator) }}
+
+{%- endmacro -%}

macros/internal/metadata_processing/escape_column_names.sql

@@ -1,6 +1,8 @@
 {%- macro escape_column_names(columns=none) -%}
 
-{# Different platforms use different escape characters, the default below is for Snowflake which uses double quotes #}
+    {%- if dbtvault.is_list(columns) -%}
+        {%- set columns = dbtvault.expand_column_list(columns) -%}
+    {%- endif -%}
 
     {%- if dbtvault.is_something(columns) -%}
 
@@ -64,31 +66,31 @@
 
     {%- endif -%}
 
-{%- if columns is none -%}
+    {%- if columns is none -%}
 
-    {%- do return(none) -%}
+        {%- do return(none) -%}
 
-{%- elif columns == [] -%}
+    {%- elif columns == [] -%}
 
-    {%- do return([]) -%}
+        {%- do return([]) -%}
 
-{%- elif columns == {} -%}
+    {%- elif columns == {} -%}
 
-    {%- do return({}) -%}
+        {%- do return({}) -%}
 
-{%- elif columns is string -%}
+    {%- elif columns is string -%}
 
-    {%- do return(col_string) -%}
+        {%- do return(col_string) -%}
 
-{%- elif dbtvault.is_list(columns) -%}
+    {%- elif dbtvault.is_list(columns) -%}
 
-    {%- do return(col_list) -%}
+        {%- do return(col_list) -%}
 
-{%- elif columns is mapping -%}
+    {%- elif columns is mapping -%}
 
-    {%- do return(col_mapping) -%}
+        {%- do return(col_mapping) -%}
 
-{%- endif -%}
+    {%- endif -%}
 
 {%- endmacro -%}
 
@@ -101,33 +103,44 @@
 
 {%- macro default__escape_column_name(column) -%}
 
-    {%- set escape_char_left  = var('escape_char_left',  '"') -%}
-    {%- set escape_char_right = var('escape_char_right', '"') -%}
+    {# Do not escape a constant (single quoted) value #}
+    {%- if column | first == "'" and column | last == "'" -%}
+        {%- set escaped_column_name = column -%}
+    {%- else -%}
+        {%- set escape_char_default_left, escape_char_default_right = dbtvault.get_escape_characters() -%}
+        {%- set escape_char_left  = var('escape_char_left',  escape_char_default_left) -%}
+        {%- set escape_char_right = var('escape_char_right', escape_char_default_right) -%}
 
-    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+        {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+    {%- endif -%}
 
     {%- do return(escaped_column_name) -%}
 
 {%- endmacro -%}
 
-{%- macro sqlserver__escape_column_name(column) -%}
-
-    {%- set escape_char_left  = var('escape_char_left',  '"') -%}
-    {%- set escape_char_right = var('escape_char_right', '"') -%}
+{% macro get_escape_characters() -%}
 
-    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+    {% do return(adapter.dispatch('get_escape_characters', 'dbtvault')()) -%}
 
-    {%- do return(escaped_column_name) -%}
-
-{%- endmacro -%}
+{%- endmacro %}
 
-{%- macro bigquery__escape_column_name(column) -%}
+{%- macro snowflake__get_escape_characters() %}
+    {%- do return (('"', '"')) -%}
+{%- endmacro %}
 
-    {%- set escape_char_left  = var('escape_char_left',  '`') -%}
-    {%- set escape_char_right = var('escape_char_right', '`') -%}
+{%- macro bigquery__get_escape_characters() %}
+    {%- do return (('`', '`')) -%}
+{%- endmacro %}
 
-    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+{%- macro sqlserver__get_escape_characters() %}
+    {%- do return (('"', '"')) -%}
+{%- endmacro %}
 
-    {%- do return(escaped_column_name) -%}
+{%- macro databricks__get_escape_characters() %}
+    {%- do return (('`', '`')) -%}
+{%- endmacro %}
 
-{%- endmacro -%}
+{%- macro postgres__get_escape_characters() %}
+    {#- DO NOT QUOTE FOR NOW. Postgres has a "feature" which froces explicit casing and breaks the SQL-92 standard -#}
+    {%- do return (('', '')) -%}
+{%- endmacro %}

macros/internal/metadata_processing/expand_column_list.sql

@@ -1,9 +1,7 @@
 {%- macro expand_column_list(columns=none) -%}
 
 {%- if not columns -%}
-    {%- if execute -%}
-         {{- exceptions.raise_compiler_error("Expected a list of columns, got: " ~ columns) -}}
-    {%- endif -%}
+    {%- do return([]) -%}
 {%- endif -%}
 
 {%- set col_list = [] -%}
@@ -49,4 +47,4 @@
 
 {%- do return(col_list) -%}
 
-{%- endmacro -%}
+{%- endmacro -%}

macros/internal/metadata_processing/process_payload_column_excludes.sql

@@ -0,0 +1,38 @@
+{%- macro process_payload_column_excludes(src_pk, src_hashdiff, src_payload, src_extra_columns,
+                                          src_eff, src_ldts, src_source, source_model) -%}
+
+    {%- if src_payload is not mapping -%}
+        {%- do return(src_payload) -%}
+    {%- endif -%}
+
+    {%- set source_model_cols = adapter.get_columns_in_relation(ref(source_model)) -%}
+    {%- set columns_in_metadata = dbtvault.expand_column_list(columns=[src_pk, src_hashdiff,
+                                                                       src_payload, src_extra_columns,
+                                                                       src_eff, src_ldts, src_source]) | map('lower') | list -%}
+
+    {%- set payload_cols = [] -%}
+    {%- for col in source_model_cols -%}
+        {%- if col.column | lower not in columns_in_metadata -%}
+            {%- do payload_cols.append(col.column) -%}
+        {%- endif -%}
+    {%- endfor -%}
+
+    {%- if 'exclude_columns' in src_payload.keys() -%}
+        {%- set table_excludes_columns = src_payload.exclude_columns -%}
+
+        {%- if table_excludes_columns -%}
+
+            {%- set excluded_payload = [] -%}
+            {%- set exclude_columns_list = src_payload.columns | map('lower') | list -%}
+
+            {%- for col in payload_cols -%}
+               {%- if col | lower not in exclude_columns_list -%}
+                   {%- do excluded_payload.append(col) -%}
+               {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+    {%- endif -%}
+
+    {%- do return(excluded_payload) -%}
+
+{%- endmacro -%}

macros/materialisations/drop_temporary.sql

@@ -0,0 +1,14 @@
+{% macro drop_temporary_special(tmp_relation) %}
+    {# In databricks and sqlserver a temporary view/table can only be dropped by #}
+    {# the connection or session that created it so drop it now before the commit below closes this session #}
+
+    {%- set drop_query_name = 'DROP_QUERY-' ~ i -%}
+    {% call statement(drop_query_name, fetch_result=True) -%}
+        {% if target.type == 'databricks' %}
+            DROP VIEW {{ tmp_relation }};
+        {% elif target.type == 'sqlserver' %}
+            DROP TABLE {{ tmp_relation }};
+        {% endif %}
+    {%- endcall %}
+
+{% endmacro %}

macros/materialisations/mat_is_checks.sql

@@ -17,7 +17,7 @@
             {{ return(relation is not none
                       and relation.type == 'table'
                       and model.config.materialized == 'vault_insert_by_period'
-                      and not flags.FULL_REFRESH) }}
+                      and not should_full_refresh()) }}
     {% endif %}
 {% endmacro %}
 
@@ -33,7 +33,7 @@
             {{ return(relation is not none
                       and relation.type == 'table'
                       and model.config.materialized == 'vault_insert_by_rank'
-                      and not flags.FULL_REFRESH) }}
+                      and not should_full_refresh()) }}
     {% endif %}
 {% endmacro %}
 
@@ -49,7 +49,7 @@
             {{ return(relation is not none
                       and relation.type == 'table'
                       and model.config.materialized == 'bridge_incremental'
-                      and not flags.FULL_REFRESH) }}
+                      and not should_full_refresh()) }}
     {% endif %}
 {% endmacro %}
 
@@ -65,6 +65,6 @@
             {{ return(relation is not none
                       and relation.type == 'table'
                       and model.config.materialized == 'pit_incremental'
-                      and not flags.FULL_REFRESH) }}
+                      and not should_full_refresh()) }}
     {% endif %}
 {% endmacro %}