Add Fabric support

macros/internal/metadata_processing/concat_ws.sql

@@ -31,3 +31,10 @@ CONCAT(
     {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }}
 
 {%- endmacro -%}
+
+
+{%- macro fabric__concat_ws(string_list, separator="||") -%}
+
+    {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }}
+
+{%- endmacro -%}

macros/internal/metadata_processing/get_escape_characters.sql

@@ -44,6 +44,10 @@
     {%- do return (('"', '"')) -%}
 {%- endmacro %}
 
+{%- macro fabric__get_escape_characters() %}
+    {%- do return (('"', '"')) -%}
+{%- endmacro %}
+
 {%- macro databricks__get_escape_characters() %}
     {%- do return (('`', '`')) -%}
 {%- endmacro %}

macros/materialisations/period_mat_helpers/check_datediff.sql

@@ -44,4 +44,17 @@
 
     {% do return(num_periods) %}
 
+{% endmacro %}
+
+{% macro fabric__check_num_periods(start_date, stop_date, period) %}
+
+    {% set num_periods_check_sql %}
+    SELECT DATEDIFF_BIG({{ period }}, CAST('{{ start_date }}' AS DATETIME2),
+        CAST(NULLIF('{{ stop_date | lower }}', 'none') AS DATETIME2)) AS NUM_PERIODS
+    {% endset %}
+    {% set num_periods_dict = automate_dv.get_query_results_as_dict(num_periods_check_sql) %}
+    {% set num_periods = num_periods_dict['NUM_PERIODS'][0] | int %}
+
+    {% do return(num_periods) %}
+
 {% endmacro %}

macros/materialisations/period_mat_helpers/get_period_boundaries.sql

@@ -118,6 +118,42 @@
 {%- endmacro %}
 
 
+{% macro fabric__get_period_boundaries(target_relation, timestamp_field, start_date, stop_date, period) -%}
+    {%- if period is in ['microsecond', 'millisecond', 'second'] -%}
+        {{ automate_dv.datepart_too_small_error(period=period) }}
+    {%- endif -%}
+
+    {#  MSSQL cannot CAST datetime2 strings with more than 7 decimal places #}
+    {% set start_date = start_date[0:27] %}
+    {% set stop_date = stop_date[0:27] %}
+    {%- set datepart = period -%}
+    {%- set from_date_or_timestamp = "CAST(NULLIF('{}','none') AS DATETIME2)".format(stop_date | lower) %}
+
+    {% set period_boundary_sql -%}
+        WITH period_data AS (
+           SELECT
+                CAST(COALESCE(MAX({{ timestamp_field }}), CAST('{{ start_date }}' AS DATETIME2)) AS DATETIME2) AS start_timestamp,
+                CAST(COALESCE({{ automate_dv.timestamp_add(datepart, interval, from_date_or_timestamp) }},
+                         {{ current_timestamp() }} ) AS DATETIME2) AS stop_timestamp
+            FROM {{ target_relation }}
+        )
+        SELECT
+            start_timestamp,
+            stop_timestamp,
+            {{ datediff('start_timestamp', 'stop_timestamp', period) }} + 1 AS num_periods
+        FROM period_data
+    {%- endset %}
+
+    {% set period_boundaries_dict = automate_dv.get_query_results_as_dict(period_boundary_sql) %}
+
+    {% set period_boundaries = {'start_timestamp': period_boundaries_dict['START_TIMESTAMP'][0] | string,
+                                'stop_timestamp': period_boundaries_dict['STOP_TIMESTAMP'][0] | string,
+                                'num_periods': period_boundaries_dict['NUM_PERIODS'][0] | int} %}
+
+    {% do return(period_boundaries) %}
+{%- endmacro %}
+
+
 {% macro databricks__get_period_boundaries(target_relation, timestamp_field, start_date, stop_date, period) -%}
 
     {%- set from_date_or_timestamp = "NULLIF('{}','none')::TIMESTAMP".format(stop_date | lower) -%}

macros/materialisations/period_mat_helpers/get_period_filter_sql.sql

@@ -47,6 +47,21 @@
 
 
 
+
+{% macro fabric__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
+    {%- set filtered_sql = {'sql': base_sql} -%}
+
+    {%- do filtered_sql.update({'sql': automate_dv.replace_placeholder_with_period_filter(core_sql=filtered_sql.sql,
+                                                                                          timestamp_field=timestamp_field,
+                                                                                          start_timestamp=start_timestamp,
+                                                                                          stop_timestamp=stop_timestamp,
+                                                                                          offset=offset, period=period)}) -%}
+    {# MSSQL does not allow CTEs in a subquery #}
+    {{ filtered_sql.sql }}
+{%- endmacro %}
+
+
+
 {% macro postgres__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
 
     {%- set filtered_sql = {'sql': base_sql} -%}

macros/materialisations/period_mat_helpers/get_period_of_load.sql

@@ -64,6 +64,22 @@
 {%- endmacro -%}
 
 
+{%- macro fabric__get_period_of_load(period, offset, start_timestamp) -%}
+    {#  MSSQL cannot CAST datetime2 strings with more than 7 decimal places #}
+    {% set start_timestamp_mssql = start_timestamp[0:23] %}
+
+    {% set period_of_load_sql -%}
+        SELECT DATEADD({{ period }}, DATEDIFF({{period}}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0) AS period_of_load
+    {%- endset %}
+
+    {% set period_of_load_dict = automate_dv.get_query_results_as_dict(period_of_load_sql) %}
+
+    {% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %}
+
+    {% do return(period_of_load) %}
+{%- endmacro -%}
+
+
 {%- macro databricks__get_period_of_load(period, offset, start_timestamp) -%}
     {% do return(automate_dv.default__get_period_of_load(period=period, offset=offset, start_timestamp=start_timestamp)) %}
 {%- endmacro -%}

macros/materialisations/period_mat_helpers/replace_placeholder_with_period_filter.sql

@@ -74,6 +74,28 @@
 {% endmacro %}
 
 
+{% macro fabric__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %}
+    {%- if period is in ['microsecond', 'millisecond', 'second'] -%}
+        {{ automate_dv.sqlserver_datepart_too_small_error(period=period) }}
+    {%- endif -%}
+
+    {#  MSSQL cannot CAST datetime2 strings with more than 7 decimal places #}
+    {% set start_timestamp_mssql = start_timestamp[0:27] %}
+
+    {%- set period_filter -%}
+    (
+        CAST({{ timestamp_field }} AS DATETIME2) >= DATEADD({{ period }}, DATEDIFF({{ period }}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0)
+            AND CAST({{ timestamp_field }} AS DATETIME2) < DATEADD({{ period }}, 1, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2)))
+            AND (CAST({{ timestamp_field }} AS DATETIME2) >= CAST('{{ start_timestamp_mssql }}' AS DATETIME2))
+    )
+    {%- endset -%}
+
+    {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%}
+
+    {% do return(filtered_sql) %}
+{% endmacro %}
+
+
 {% macro postgres__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %}
 
     {%- set period_filter -%}

macros/materialisations/rank_mat_helpers/replace_placeholder_with_rank_filter.sql

@@ -42,6 +42,20 @@
 
 
 
+{% macro fabric__replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) %}
+
+    {%- set rank_filter -%}
+        CAST({{ rank_column }} AS INT) = CAST({{ rank_iteration }} AS INT)
+    {%- endset -%}
+
+    {%- set filtered_sql = core_sql | replace("__RANK_FILTER__", rank_filter) -%}
+
+    {% do return(filtered_sql) %}
+{% endmacro %}
+
+
+
+
 {% macro bigquery__replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) %}
     {%- set rank_filter -%}
         CAST({{ rank_column }} AS INTEGER) = CAST({{ rank_iteration }} AS INTEGER)

macros/staging/null_columns.sql

@@ -80,6 +80,13 @@
 
 {%- endmacro -%}
 
+{%- macro fabric__null_column_sql(col_name, default_value) -%}
+
+    {{ col_name }} AS {{ col_name ~ "_ORIGINAL" }},
+    ISNULL({{ col_name }}, '{{ default_value }}') AS {{ col_name }}
+
+{%- endmacro -%}
+
 {%- macro postgres__null_column_sql(col_name, default_value) -%}
 
     {{ col_name }} AS {{ col_name ~ "_ORIGINAL" }},

macros/supporting/casting/cast_binary.sql

@@ -33,6 +33,19 @@
 
 {%- endmacro -%}
 
+
+{%- macro fabric__cast_binary(column_str, alias=none, quote=true) -%}
+
+    {%- if quote -%}
+        CONVERT({{ automate_dv.type_binary() }}, '{{ column_str }}', 2)
+    {%- else -%}
+        CONVERT({{ automate_dv.type_binary() }}, {{ column_str }}, 2)
+    {%- endif -%}
+
+    {% if alias %} AS {{ alias }} {%- endif %}
+
+{%- endmacro -%}
+
 {%- macro bigquery__cast_binary(column_str, alias=none, quote=true) -%}
 
     {{ automate_dv.default__cast_binary(column_str=column_str, alias=alias, quote=quote) }}

macros/supporting/casting/cast_date.sql

@@ -38,6 +38,20 @@
 {%- endmacro -%}
 
 
+{%- macro fabric__cast_date(column_str, as_string=false, alias=none) -%}
+
+    {%- if not as_string -%}
+        CONVERT(DATE, {{ column_str }})
+    {%- else -%}
+        CONVERT(DATE, '{{ column_str }}')
+    {%- endif -%}
+
+    {%- if alias %} AS {{ alias }} {%- endif %}
+
+
+{%- endmacro -%}
+
+
 {%- macro bigquery__cast_date(column_str, as_string=false, alias=none) -%}
 
     {%- if not as_string -%}

macros/supporting/casting/cast_datetime.sql

@@ -40,6 +40,15 @@
 {%- endmacro -%}
 
 
+{%- macro fabric__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%}
+
+    CONVERT(DATETIME2, {{ column_str }})
+
+    {%- if alias %} AS {{ alias }} {%- endif %}
+
+{%- endmacro -%}
+
+
 {%- macro bigquery__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%}
 
     {%- if date_type == 'timestamp' -%}

macros/supporting/data_types/type_binary.sql

@@ -29,4 +29,8 @@
 
 {%- macro databricks__type_binary() -%}
     STRING
+{%- endmacro -%}
+
+{%- macro fabric__type_binary() -%}
+    UNIQUEIDENTIFIER
 {%- endmacro -%}

macros/supporting/data_types/type_string.sql

@@ -19,6 +19,10 @@
     VARCHAR
 {%- endmacro -%}
 
+{%- macro fabric__type_string(is_hash, char_length) -%}
+    VARCHAR
+{%- endmacro -%}
+
 {%- macro databricks__type_string(is_hash=false, char_length=255) -%}
     {%- if is_hash -%}
         {%- if var('hash', 'MD5') | lower == 'md5' -%}

macros/supporting/data_types/type_timestamp.sql

@@ -13,4 +13,8 @@
 
 {%- macro sqlserver__type_timestamp() -%}
     DATETIME2
+{%- endmacro -%}
+
+{%- macro fabric__type_timestamp() -%}
+    DATETIME2
 {%- endmacro -%}

macros/supporting/ghost_records/binary_ghost.sql

@@ -33,3 +33,17 @@
 
     {%- if alias %} AS {{ alias }} {%- endif -%}
 {%- endmacro -%}
+
+{%- macro fabric__binary_ghost(alias, hash) -%}
+    {%- if hash | lower == 'md5' -%}
+        CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(16)), 16) AS BINARY(16))
+	{%- elif hash | lower == 'sha' -%}
+        CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(32)), 32) AS BINARY(32))
+    {%- elif hash | lower == 'sha1' -%}
+        CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(20)), 20) AS BINARY(20))
+    {%- else -%}
+        CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY(16)), 16) AS BINARY(16))
+    {%- endif -%}
+
+    {%- if alias %} AS {{ alias }} {%- endif -%}
+{%- endmacro -%}

macros/supporting/ghost_records/null_ghost.sql

@@ -21,4 +21,8 @@
 
 {%- macro sqlserver__null_ghost(datatype, alias) -%}
     {{ automate_dv.bigquery__null_ghost(datatype, alias) }}
+{%- endmacro -%}
+
+{%- macro fabric__null_ghost(datatype, alias) -%}
+    {{ automate_dv.bigquery__null_ghost(datatype, alias) }}
 {%- endmacro -%}

macros/supporting/hash.sql

@@ -107,6 +107,13 @@
 {%- endmacro -%}
 
 
+{%- macro fabric__hash(columns, alias, is_hashdiff, columns_to_escape) -%}
+
+    {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }}
+
+{%- endmacro -%}
+
+
 {%- macro postgres__hash(columns, alias, is_hashdiff, columns_to_escape) -%}
 
     {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }}

macros/supporting/hash_components/null_expression.sql

@@ -45,4 +45,14 @@
 
     {% do return(column_expression) %}
 
+{%- endmacro -%}
+
+{%- macro fabric__null_expression(standardise, column_str, null_placeholder_string) -%}
+
+    {%- set column_expression -%}
+        ISNULL({{ standardise | replace('[EXPRESSION]', column_str) }}, '{{ null_placeholder_string }}')
+    {%- endset -%}
+
+    {% do return(column_expression) %}
+
 {%- endmacro -%}

macros/supporting/hash_components/select_hash_alg.sql

@@ -49,6 +49,12 @@
 
 {% endmacro %}
 
+{% macro fabric__hash_alg_md5() -%}
+
+    {% do return(automate_dv.cast_binary("HASHBYTES('MD5', [HASH_STRING_PLACEHOLDER])", quote=false)) %}
+
+{% endmacro %}
+
 {% macro postgres__hash_alg_md5() -%}
 
     {% do return("DECODE(MD5([HASH_STRING_PLACEHOLDER]), 'hex')") %}
@@ -89,6 +95,12 @@
 
 {% endmacro %}
 
+{% macro fabric__hash_alg_sha256() -%}
+
+    {% do return(automate_dv.cast_binary("HASHBYTES('SHA2_256', [HASH_STRING_PLACEHOLDER])", quote=false)) %}
+
+{% endmacro %}
+
 {% macro postgres__hash_alg_sha256() -%}
     {#- * MD5 is simple function call to md5(val) -#}
     {#- * SHA256 needs input cast to BYTEA and then its BYTEA result encoded as hex text output -#}
@@ -131,6 +143,12 @@
 
 {% endmacro %}
 
+{% macro fabric__hash_alg_sha1() -%}
+
+    {% do return(automate_dv.cast_binary("HASHBYTES('SHA1', [HASH_STRING_PLACEHOLDER])", quote=false)) %}
+
+{% endmacro %}
+
 {% macro postgres__hash_alg_sha1() -%}
 
     {%- do exceptions.warn("Configured hash (SHA-1) is not supported on Postgres.