dbtvault 0.8

README.md

@@ -1,24 +1,17 @@
-<p align="center">
+<div align="center">
   <img src="https://user-images.githubusercontent.com/25080503/65772647-89525700-e132-11e9-80ff-12ad30a25466.png" alt="dbtvault">
-</p>
 
-<p align="center">
-  <a href="https://dbtvault.readthedocs.io/en/stable"><img
-    src="https://img.shields.io/badge/docs-stable-blue" 
-    alt="Documentation"
-  /></a>
-  <a href="https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA"><img
-    src="https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack" 
-    alt="Join our slack"
-  /></a>
-</p>
+  [![Documentation Status](https://readthedocs.org/projects/dbtvault/badge/?version=stable)](https://dbtvault.readthedocs.io/en/stable/?badge=stable)
+  [![Slack](https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack)](https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA)
+
+</div>
 
 
 [Changelog and past doc versions](https://dbtvault.readthedocs.io/en/latest/changelog/stable)
 
 # dbtvault by [Datavault](https://www.data-vault.co.uk)
 
-Build your own Data Vault data warehouse! dbtvault is a free to use dbt package that generates & executes the ETL you need to run a Data Vault 2.0 Data Warehouse on a Snowflake database.
+Build your own Data Vault data warehouse! dbtvault is a free to use dbt package that generates & executes the ETL you need to run a Data Vault 2.0 Data Warehouse on your data platform.
 
 What does dbtvault offer?
 - productivity gains, fewer errors
@@ -37,9 +30,9 @@ Learn quickly with our worked example:
 
 - [Project Repository](https://github.com/Datavault-UK/snowflakeDemo)
 
-## Supported databases:
+## Supported platforms:
 
-- [snowflake](https://www.snowflake.com/about/)
+[Platform support matrix](https://dbtvault.readthedocs.io/en/latest/macros/#platform-support)
 
 ## Installation
 
@@ -56,7 +49,7 @@ or [read the docs](https://docs.getdbt.com/docs/building-a-dbt-project/package-m
 # Configure model
 {{- config(...)                          -}}
 
-# Set metadata
+# Provide metadata
 {%- set src_pk = ...                     -%}
 ...
 

macros/internal/is_checks.sql → macros/internal/helpers/is_checks.sql

@@ -12,6 +12,8 @@
 
 {%- endmacro -%}
 
+
+
 {%- macro is_nothing(obj) -%}
 
     {%- if obj is none or obj is undefined or not obj -%}
@@ -22,6 +24,8 @@
 
 {%- endmacro -%}
 
+
+
 {%- macro is_something(obj) -%}
 
     {%- if obj is not none and obj is defined and obj -%}
@@ -30,4 +34,20 @@
         {%- do return(false) -%}
     {%- endif -%}
 
+{%- endmacro -%}
+
+
+
+{%- macro is_expression(obj) -%}
+
+    {%- if obj is string -%}
+        {%- if (obj | first == "'" and obj | last == "'") or ("(" in obj and ")" in obj) or "::" in obj -%}
+            {%- do return(true) -%}
+        {%- else -%}
+            {%- do return(false) -%}
+        {%- endif -%}
+    {%- else -%}
+        {%- do return(false) -%}
+    {%- endif -%}
+
 {%- endmacro -%}

macros/internal/as_constant.sql → macros/internal/metadata_processing/as_constant.sql

@@ -14,8 +14,16 @@
 
         {%- else -%}
 
-            {{- return(column_str) -}}
-
+            {%- if dbtvault.is_expression(column_str) -%}
+
+                {{- return(column_str) -}}
+
+            {%- else -%}
+
+                {{- return(dbtvault.escape_column_names(column_str)) -}}
+
+            {%- endif -%}
+
         {%- endif -%}
     {%- else -%}
         {%- if execute -%}

macros/internal/metadata_processing/concat_ws.sql

@@ -0,0 +1,22 @@
+{%- macro concat_ws(string_list, separator="||") -%}
+
+    {{- adapter.dispatch('concat_ws', 'dbtvault')(string_list=string_list, separator=separator) -}}
+
+{%- endmacro %}
+
+{%- macro default__concat_ws(string_list, separator="||") -%}
+
+    {{  "CONCAT_WS('" ~ separator ~ "', " ~ string_list | join(", ") ~ ")" }}
+
+{%- endmacro -%}
+
+{%- macro bigquery__concat_ws(string_list, separator="||") -%}
+
+    {{- 'CONCAT(' -}}
+    {%- for str in string_list -%}
+        {{- "{}".format(str) -}}
+        {{- ",'{}',".format(separator) if not loop.last -}}
+    {%- endfor -%}
+    {{- '\n)' -}}
+
+{%- endmacro -%}

macros/internal/metadata_processing/escape_column_names.sql

@@ -0,0 +1,133 @@
+{%- macro escape_column_names(columns=none) -%}
+
+{# Different platforms use different escape characters, the default below is for Snowflake which uses double quotes #}
+
+    {%- if dbtvault.is_something(columns) -%}
+
+        {%- set col_string = '' -%}
+        {%- set col_list = [] -%}
+        {%- set col_mapping = {} -%}
+
+        {%- if columns is string -%}
+
+            {%- set col_string = dbtvault.escape_column_name(columns) -%}
+
+        {%- elif dbtvault.is_list(columns) -%}
+
+            {%- for col in columns -%}
+
+                {%- if col is string -%}
+
+                    {%- set escaped_col = dbtvault.escape_column_name(col) -%}
+
+                    {%- do col_list.append(escaped_col) -%}
+
+                {%- else -%}
+
+                    {%- if execute -%}
+                        {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string.") -}}
+                    {%- endif -%}
+
+                {%- endif -%}
+
+            {%- endfor -%}
+
+        {%- elif columns is mapping -%}
+
+            {%- if columns['source_column'] and columns['alias'] -%}
+
+                {%- set escaped_source_col = dbtvault.escape_column_name(columns['source_column']) -%}
+                {%- set escaped_alias_col = dbtvault.escape_column_name(columns['alias']) -%}
+                {%- set col_mapping = {"source_column": escaped_source_col, "alias": escaped_alias_col} -%}
+
+            {%- else -%}
+
+                {%- if execute -%}
+                    {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string, a list of strings, or a dictionary of hashdiff metadata.") -}}
+                {%- endif %}
+
+            {%- endif -%}
+
+        {%- else -%}
+
+            {%- if execute -%}
+                {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string, a list of strings, or a dictionary of hashdiff metadata.") -}}
+            {%- endif %}
+
+        {%- endif -%}
+
+    {%- elif columns == '' -%}
+
+        {%- if execute -%}
+            {{- exceptions.raise_compiler_error("Expected a column name or a list of column names, got an empty string") -}}
+        {%- endif -%}
+
+    {%- endif -%}
+
+{%- if columns is none -%}
+
+    {%- do return(none) -%}
+
+{%- elif columns == [] -%}
+
+    {%- do return([]) -%}
+
+{%- elif columns == {} -%}
+
+    {%- do return({}) -%}
+
+{%- elif columns is string -%}
+
+    {%- do return(col_string) -%}
+
+{%- elif dbtvault.is_list(columns) -%}
+
+    {%- do return(col_list) -%}
+
+{%- elif columns is mapping -%}
+
+    {%- do return(col_mapping) -%}
+
+{%- endif -%}
+
+{%- endmacro -%}
+
+
+{%- macro escape_column_name(column) -%}
+
+    {{- adapter.dispatch('escape_column_name', 'dbtvault')(column=column) -}}
+
+{%- endmacro %}
+
+{%- macro default__escape_column_name(column) -%}
+
+    {%- set escape_char_left  = var('escape_char_left',  '"') -%}
+    {%- set escape_char_right = var('escape_char_right', '"') -%}
+
+    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+
+    {%- do return(escaped_column_name) -%}
+
+{%- endmacro -%}
+
+{%- macro sqlserver__escape_column_name(column) -%}
+
+    {%- set escape_char_left  = var('escape_char_left',  '"') -%}
+    {%- set escape_char_right = var('escape_char_right', '"') -%}
+
+    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+
+    {%- do return(escaped_column_name) -%}
+
+{%- endmacro -%}
+
+{%- macro bigquery__escape_column_name(column) -%}
+
+    {%- set escape_char_left  = var('escape_char_left',  '`') -%}
+    {%- set escape_char_right = var('escape_char_right', '`') -%}
+
+    {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%}
+
+    {%- do return(escaped_column_name) -%}
+
+{%- endmacro -%}

macros/materialisations/incremental_bridge_materialization.sql

@@ -1,53 +1,57 @@
-{% materialization bridge_incremental, default -%}
+{%- materialization bridge_incremental, default -%}
 
-  {% set full_refresh_mode = flags.FULL_REFRESH %}
+  {%- set full_refresh_mode = should_full_refresh() -%}
 
-  {% set target_relation = this %}
-  {% set existing_relation = load_relation(this) %}
-  {% set tmp_relation = make_temp_relation(this) %}
+  {% if target.type == "sqlserver" %}
+      {%- set target_relation = this.incorporate(type='table') -%}
+  {%  else %}
+      {%- set target_relation = this -%}
+  {% endif %}
+  {%- set existing_relation = load_relation(this) -%}
+  {%- set tmp_relation = make_temp_relation(target_relation) -%}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
-  {% set to_drop = [] %}
-  {% if existing_relation is none %}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
-  {% elif existing_relation.is_view or full_refresh_mode %}
+  {%- set to_drop = [] -%}
+  {%- if existing_relation is none -%}
+      {%- set build_sql = create_table_as(False, target_relation, sql) -%}
+  {%- elif existing_relation.is_view or full_refresh_mode -%}
       {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
-      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
-      {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
-      {% do adapter.drop_relation(backup_relation) %}
-
-      {% do adapter.rename_relation(target_relation, backup_relation) %}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
-      {% do to_drop.append(backup_relation) %}
-  {% else %}
-
-      {% set tmp_relation = make_temp_relation(target_relation) %}
-      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
-      {% do adapter.expand_target_column_types(
+      {%- set backup_identifier = existing_relation.identifier ~ "__dbt_backup" -%}
+      {%- set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) -%}
+      {%- do adapter.drop_relation(backup_relation) -%}
+
+      {%- do adapter.rename_relation(target_relation, backup_relation) -%}
+      {%- set build_sql = create_table_as(False, target_relation, sql) -%}
+      {%- do to_drop.append(backup_relation) -%}
+  {%- else -%}
+
+      {%- set tmp_relation = make_temp_relation(target_relation) -%}
+      {%- do run_query(create_table_as(True, tmp_relation, sql)) -%}
+      {%- do adapter.expand_target_column_types(
              from_relation=tmp_relation,
-             to_relation=target_relation) %}
-      {% set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) %}
-{% endif %}
+             to_relation=target_relation) -%}
+      {%- set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) -%}
+{%- endif -%}
 
-  {% call statement("main") %}
+  {%- call statement("main") -%}
       {{ build_sql }}
-  {% endcall %}
+  {%- endcall -%}
 
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here
-  {% do adapter.commit() %}
+  {%- do adapter.commit() -%}
 
-  {% for rel in to_drop %}
-      {% do adapter.drop_relation(rel) %}
-  {% endfor %}
+  {%- for rel in to_drop -%}
+      {%- do adapter.drop_relation(rel) -%}
+  {%- endfor -%}
 
   {{ run_hooks(post_hooks, inside_transaction=False) }}
 
   {{ return({'relations': [target_relation]}) }}
 
-{%- endmaterialization %}
+{%- endmaterialization -%}