From 0fac4411fbd32c85454b90540327afac71f6796f Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 10:59:10 +0200 Subject: [PATCH 01/18] Added tox.ini and profiles for tests --- integration_test_project/profiles.yml | 11 +++++++++++ tox.ini | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..c5188699 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,3 +52,14 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 + sqlserver: + type: sqlserver + driver: 'ODBC Driver 18 for SQL Server' + server: localhost + port: 1433 + database: dbt_artifact_integrationtests + schema: dbo + windows_login: False + trust_cert: True + user: dbt + password: "123" diff --git a/tox.ini b/tox.ini index aee9fc2b..480d7821 100644 --- a/tox.ini +++ b/tox.ini @@ -353,4 +353,26 @@ commands = dbt deps dbt build --target postgres +[testenv:integration_sqlserver] +changedir = integration_test_project +deps = dbt-sqlserver~=1.7.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver +[testenv:integration_sqlserver_1_4_3] +changedir = integration_test_project +deps = dbt-sqlserver~=1.4.3 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + +[testenv:integration_sqlserver_1_7_4] +changedir = integration_test_project +deps = dbt-sqlserver~=1.7.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver From 8d7efeb5e562c7b2ebb328b99198a976cfbb9fa4 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 10:59:24 +0200 Subject: [PATCH 02/18] Added SQL Server support to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 93c25f40..4569b8ed 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The package currently supports - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: - Postgres :white_check_mark: +- SQL Server :white_check_mark: Models included: From 37c3624f33eff4d86f846eb44d7039e3da26e152 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 10:59:40 +0200 Subject: [PATCH 03/18] Added column-level support exclusion for SQL Server --- dbt_project.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 684efce6..7d296b51 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -18,5 +18,6 @@ models: +on_schema_change: append_new_columns +full_refresh: false +persist_docs: - # Databricks doesn't offer column-level support for persisting docs - columns: '{{ target.name != "databricks" }}' + # Databricks and SQL Server don't offer column-level support for persisting docs + columns: '{{ target.name != "databricks" and target.name != "sqlserver" }}' + +as_columnstore: False From 39e47c7f150cef173faf775739a0577bc4d08b1e Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 11:00:05 +0200 Subject: [PATCH 04/18] Added required alias to subquery for SQL Server --- integration_test_project/tests/singular_test.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index a1e49e8c..35652754 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1,2 @@ -select 1 as failures from (select 2) as foo where 1 = 2 +select 1 as failures from (select 2 as two) as foo where 1 = 2 + From a85d3e88beafc642438f5b3aecad7cf1e1dd4e6f Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 11:00:29 +0200 Subject: [PATCH 05/18] Added relation and columns exclusion for SQL Server --- integration_test_project/dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index dba89a50..aae6f8df 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -25,8 +25,8 @@ vars: models: +persist_docs: - relation: true - columns: true + relation: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this + columns: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this seeds: +quote_columns: false From 797931889cd314eecde57f788fc18bb08f8f39c2 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 11:01:53 +0200 Subject: [PATCH 06/18] Added SQL Server specific uploads --- .../upload_exposures.sql | 50 ++++++++-- .../upload_invocations.sql | 91 +++++++++++++++++-- .../upload_model_executions.sql | 61 +++++++++++-- .../upload_models.sql | 55 +++++++++-- .../upload_seed_executions.sql | 61 +++++++++++-- .../upload_seeds.sql | 48 ++++++++-- .../upload_snapshot_executions.sql | 61 +++++++++++-- .../upload_snapshots.sql | 52 +++++++++-- .../upload_sources.sql | 48 ++++++++-- .../upload_test_executions.sql | 61 +++++++++++-- .../upload_tests.sql | 45 +++++++-- 11 files changed, 546 insertions(+), 87 deletions(-) diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 9f0ec5d3..e097be7e 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -1,5 +1,5 @@ {% macro upload_exposures(exposures) -%} - {{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }} + {{ return(adapter.dispatch("get_exposures_dml_sql", "dbt_artifacts")(exposures)) }} {%- endmacro %} {% macro default__get_exposures_dml_sql(exposures) -%} @@ -47,8 +47,7 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -80,8 +79,7 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -114,7 +112,45 @@ {%- endfor %} {% endset %} {{ exposure_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ exposure.unique_id | replace("'","''") }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ exposure.name | replace("'","''") }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ exposure.original_file_path }}', {# path #} + '{{ exposure.description | replace("'","''") }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ tojson(exposure.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ exposure_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 21c5574c..775861ac 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -2,24 +2,22 @@ {# Need to remove keys with results that can't be handled properly #} {# warn_error_options - returns a python object in 1.5 #} - {% if 'warn_error_options' in invocation_args_dict %} + {% if "warn_error_options" in invocation_args_dict %} {% if invocation_args_dict.warn_error_options is not string %} {% if invocation_args_dict.warn_error_options.include %} {% set include_options = invocation_args_dict.warn_error_options.include %} - {% else %} - {% set include_options = '' %} + {% else %} {% set include_options = "" %} {% endif %} {% if invocation_args_dict.warn_error_options.exclude %} {% set exclude_options = invocation_args_dict.warn_error_options.exclude %} - {% else %} - {% set exclude_options = '' %} + {% else %} {% set exclude_options = "" %} {% endif %} - {% set warn_error_options = {'include': include_options, 'exclude': exclude_options} %} - {%- do invocation_args_dict.update({'warn_error_options': warn_error_options}) %} + {% set warn_error_options = {"include": include_options, "exclude": exclude_options} %} + {%- do invocation_args_dict.update({"warn_error_options": warn_error_options}) %} {% endif %} {% endif %} - {{ return(adapter.dispatch('get_invocations_dml_sql', 'dbt_artifacts')()) }} + {{ return(adapter.dispatch("get_invocations_dml_sql", "dbt_artifacts")()) }} {%- endmacro %} {% macro default__get_invocations_dml_sql() -%} @@ -221,3 +219,80 @@ {{ invocation_values }} {% endmacro -%} + + +{% macro sqlserver__get_invocations_dml_sql() -%} + {% set invocation_values %} + select + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + nullif("11", ''), + nullif("12", ''), + nullif("13", ''), + nullif("14", ''), + nullif("15", ''), + "16", + "17", + "18", + "19" + from (values + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} + '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} + '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} + '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #} + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(env_vars_dict) }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + '{{ tojson(invocation_args_dict) | replace("'", "''") }}', {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (value | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(metadata_env) }}' {# dbt_custom_envs #} + + ) + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19") + + {% endset %} + {{ invocation_values }} + +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index bca26fea..1d5fa83b 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -1,5 +1,5 @@ {% macro upload_model_executions(models) -%} - {{ return(adapter.dispatch('get_model_executions_dml_sql', 'dbt_artifacts')(models)) }} + {{ return(adapter.dispatch("get_model_executions_dml_sql", "dbt_artifacts")(models)) }} {%- endmacro %} {% macro default__get_model_executions_dml_sql(models) -%} @@ -57,8 +57,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -99,8 +98,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -158,8 +156,7 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -199,7 +196,53 @@ {%- endfor %} {% endset %} {{ model_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ( "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" ) + + {% endset %} + {{ model_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 7570b06a..2c27daec 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -1,5 +1,5 @@ {% macro upload_models(models) -%} - {{ return(adapter.dispatch('get_models_dml_sql', 'dbt_artifacts')(models)) }} + {{ return(adapter.dispatch("get_models_dml_sql", "dbt_artifacts")(models)) }} {%- endmacro %} {% macro default__get_models_dml_sql(models) -%} @@ -51,8 +51,7 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -87,8 +86,7 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -123,7 +121,50 @@ {%- endfor %} {% endset %} {{ model_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15" + from ( values + {% for model in models -%} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model_copy.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + '{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + '{{ model_copy.original_file_path }}', {# path #} + '{{ model_copy.checksum.checksum }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + '{{ tojson(model_copy.tags) }}', {# tags #} + '{{ tojson(model_copy.config.meta) | replace("'","''") }}', {# meta #} + '{{ model_copy.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model_copy) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15") + + {% endset %} + {{ model_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 1ccbfe2a..c19a9a2f 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -1,5 +1,5 @@ {% macro upload_seed_executions(seeds) -%} - {{ return(adapter.dispatch('get_seed_executions_dml_sql', 'dbt_artifacts')(seeds)) }} + {{ return(adapter.dispatch("get_seed_executions_dml_sql", "dbt_artifacts")(seeds)) }} {%- endmacro %} {% macro default__get_seed_executions_dml_sql(seeds) -%} @@ -56,8 +56,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -97,8 +96,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -156,8 +154,7 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -213,7 +210,53 @@ {%- endfor %} {% endset %} {{ seed_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ seed_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 32e67383..2501a9e9 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -1,5 +1,5 @@ {% macro upload_seeds(seeds) -%} - {{ return(adapter.dispatch('get_seeds_dml_sql', 'dbt_artifacts')(seeds)) }} + {{ return(adapter.dispatch("get_seeds_dml_sql", "dbt_artifacts")(seeds)) }} {%- endmacro %} {% macro default__get_seeds_dml_sql(seeds) -%} @@ -43,8 +43,7 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -74,8 +73,7 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -105,7 +103,43 @@ {%- endfor %} {% endset %} {{ seed_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_seeds_dml_sql(seeds) -%} + + {% if seeds != [] %} + {% set seed_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ tojson(seed.config.meta) | replace("'","''") }}', {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + + {% endset %} + {{ seed_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index 2006b168..b0af313b 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -1,5 +1,5 @@ {% macro upload_snapshot_executions(snapshots) -%} - {{ return(adapter.dispatch('get_snapshot_executions_dml_sql', 'dbt_artifacts')(snapshots)) }} + {{ return(adapter.dispatch("get_snapshot_executions_dml_sql", "dbt_artifacts")(snapshots)) }} {%- endmacro %} {% macro default__get_snapshot_executions_dml_sql(snapshots) -%} @@ -56,8 +56,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -97,8 +96,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -156,8 +154,7 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -213,7 +210,53 @@ {%- endfor %} {% endset %} {{ snapshot_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ snapshot_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 0be6759b..e540dc0b 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -1,6 +1,6 @@ {% macro upload_snapshots(snapshots) -%} - {{ return(adapter.dispatch('get_snapshots_dml_sql', 'dbt_artifacts')(snapshots)) }} + {{ return(adapter.dispatch("get_snapshots_dml_sql", "dbt_artifacts")(snapshots)) }} {%- endmacro %} @@ -49,8 +49,7 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -82,8 +81,7 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -115,7 +113,47 @@ {%- endfor %} {% endset %} {{ snapshot_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_snapshots_dml_sql(snapshots) -%} + + {% if snapshots != [] %} + {% set snapshot_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ tojson(snapshot.config.meta) | replace("'","''") }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ snapshot_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index dbcb49e4..3d899755 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -1,5 +1,5 @@ {% macro upload_sources(sources) -%} - {{ return(adapter.dispatch('get_sources_dml_sql', 'dbt_artifacts')(sources)) }} + {{ return(adapter.dispatch("get_sources_dml_sql", "dbt_artifacts")(sources)) }} {%- endmacro %} {% macro default__get_sources_dml_sql(sources) -%} @@ -43,8 +43,7 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -74,8 +73,7 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -105,7 +103,43 @@ {%- endfor %} {% endset %} {{ source_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_sources_dml_sql(sources) -%} + + {% if sources != [] %} + {% set source_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ source.loaded_at_field | replace("'","''") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'","''") }}', {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + {% endset %} + {{ source_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index cb13288b..3af31d73 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -1,5 +1,5 @@ {% macro upload_test_executions(tests) -%} - {{ return(adapter.dispatch('get_test_executions_dml_sql', 'dbt_artifacts')(tests)) }} + {{ return(adapter.dispatch("get_test_executions_dml_sql", "dbt_artifacts")(tests)) }} {%- endmacro %} {% macro default__get_test_executions_dml_sql(tests) -%} @@ -50,8 +50,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -89,8 +88,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -144,8 +142,7 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -197,7 +194,51 @@ {%- endfor %} {% endset %} {{ test_execution_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} -{% endmacro -%} \ No newline at end of file +{% endmacro -%} + +{% macro sqlserver__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("'", "''") }}', {# message #} + '{{ tojson(test.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) AS v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13") + + {% endset %} + {{ test_execution_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + + diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index cf75e9ba..18d7171c 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -1,5 +1,5 @@ {% macro upload_tests(tests) -%} - {{ return(adapter.dispatch('get_tests_dml_sql', 'dbt_artifacts')(tests)) }} + {{ return(adapter.dispatch("get_tests_dml_sql", "dbt_artifacts")(tests)) }} {%- endmacro %} {% macro default__get_tests_dml_sql(tests) -%} @@ -37,8 +37,7 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {% endmacro -%} @@ -65,8 +64,7 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} @@ -93,7 +91,40 @@ {%- endfor %} {% endset %} {{ test_values }} - {% else %} - {{ return("") }} + {% else %} {{ return("") }} {% endif %} {%- endmacro %} + + +{% macro sqlserver__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path }}', {# test_path #} + '{{ tojson(test.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("'","''") }}' {# all_fields #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9") + {% endset %} + {{ test_values }} + {% else %} {{ return("") }} + {% endif %} +{% endmacro -%} + From a24997de261e222d53bd470a9142dcfc6890e813 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 11:02:32 +0200 Subject: [PATCH 07/18] Delimited schema and database because these are reserved keywords --- macros/migration/migrate_from_v0_to_v1.sql | 63 +++---- .../upload_results/get_column_name_lists.sql | 54 +++--- .../insert_into_metadata_table.sql | 12 ++ models/dim_dbt__current_models.sql | 163 +++++++++--------- models/dim_dbt__models.sql | 44 +++-- models/dim_dbt__seeds.sql | 38 ++-- models/dim_dbt__snapshots.sql | 42 ++--- models/dim_dbt__sources.sql | 38 ++-- models/fct_dbt__model_executions.sql | 50 +++--- models/fct_dbt__seed_executions.sql | 46 +++-- models/fct_dbt__snapshot_executions.sql | 46 +++-- models/sources/model_executions.sql | 2 +- models/sources/models.sql | 4 +- models/sources/seed_executions.sql | 2 +- models/sources/seeds.sql | 4 +- models/sources/snapshot_executions.sql | 2 +- models/sources/snapshots.sql | 4 +- models/sources/sources.sql | 4 +- models/staging/stg_dbt__model_executions.sql | 2 +- models/staging/stg_dbt__models.sql | 4 +- models/staging/stg_dbt__seed_executions.sql | 2 +- models/staging/stg_dbt__seeds.sql | 4 +- .../staging/stg_dbt__snapshot_executions.sql | 2 +- models/staging/stg_dbt__snapshots.sql | 4 +- models/staging/stg_dbt__sources.sql | 4 +- 25 files changed, 302 insertions(+), 338 deletions(-) diff --git a/macros/migration/migrate_from_v0_to_v1.sql b/macros/migration/migrate_from_v0_to_v1.sql index 63383077..b4cdc724 100644 --- a/macros/migration/migrate_from_v0_to_v1.sql +++ b/macros/migration/migrate_from_v0_to_v1.sql @@ -9,7 +9,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -34,9 +34,7 @@ {% endset %} {{ log("Migrating model_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_model_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_model_executions }} {%- endcall -%} {% set migrate_tests %} insert into {{new_database}}.{{new_schema}}.tests ( @@ -62,9 +60,7 @@ {% endset %} {{ log("Migrating tests", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_tests }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_tests }} {%- endcall -%} {% set migrate_test_executions %} insert into {{new_database}}.{{new_schema}}.test_executions ( @@ -96,22 +92,20 @@ {% endset %} {{ log("Migrating test_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_test_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_test_executions }} {%- endcall -%} {% set migrate_models %} insert into {{new_database}}.{{new_schema}}.models ( checksum, command_invocation_id, - database, + "database", depends_on_nodes, materialization, name, node_id, package_name, path, - schema, + "schema", run_started_at ) select @@ -130,20 +124,18 @@ {% endset %} {{ log("Migrating models", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_models }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_models }} {%- endcall -%} {% set migrate_seeds %} insert into {{new_database}}.{{new_schema}}.seeds ( checksum, command_invocation_id, - database, + "database", name, node_id, package_name, path, - schema, + "schema", run_started_at ) select @@ -160,9 +152,7 @@ {% endset %} {{ log("Migrating seeds", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_seeds }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_seeds }} {%- endcall -%} {% set migrate_seed_executions %} insert into {{new_database}}.{{new_schema}}.seed_executions ( @@ -173,7 +163,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -198,9 +188,7 @@ {% endset %} {{ log("Migrating seed_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_seed_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_seed_executions }} {%- endcall -%} {% set migrate_exposures %} insert into {{new_database}}.{{new_schema}}.exposures ( @@ -235,21 +223,19 @@ {% endset %} {{ log("Migrating exposures", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_exposures }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_exposures }} {%- endcall -%} {% set migrate_snapshots %} insert into {{new_database}}.{{new_schema}}.snapshots ( checksum, command_invocation_id, - database, + "database", depends_on_nodes, name, node_id, package_name, path, - schema, + "schema", strategy, run_started_at ) @@ -269,9 +255,7 @@ {% endset %} {{ log("Migrating snapshots", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_snapshots }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_snapshots }} {%- endcall -%} {% set migrate_snapshot_executions %} insert into {{new_database}}.{{new_schema}}.snapshot_executions ( @@ -282,7 +266,7 @@ node_id, query_completed_at, rows_affected, - schema, + "schema", status, thread_id, total_node_runtime, @@ -307,21 +291,19 @@ {% endset %} {{ log("Migrating snapshot_executions", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_snapshot_executions }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_snapshot_executions }} {%- endcall -%} {% set migrate_sources %} insert into {{new_database}}.{{new_schema}}.sources ( command_invocation_id, - database, + "database", freshness, identifier, loaded_at_field, loader, name, node_id, - schema, + "schema", source_name, run_started_at ) @@ -341,9 +323,8 @@ {% endset %} {{ log("Migrating sources", info=True) }} - {%- call statement(auto_begin=True) -%} - {{ migrate_sources }} - {%- endcall -%} + {%- call statement(auto_begin=True) -%} {{ migrate_sources }} {%- endcall -%} {{ log("Migration complete. You can now safely delete any data from before 1.0.0", info=True) }} {%- endmacro -%} + diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index 7911e866..37f22500 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -1,12 +1,10 @@ - {# These are the column lists used as part of the upload macros - the order here should be the same as the order in each individual `upload_dataset` macro. #} - {% macro get_column_name_list(dataset) -%} - {% if dataset == 'exposures' %} + {% if dataset == "exposures" %} ( command_invocation_id, @@ -25,7 +23,7 @@ all_results ) - {% elif dataset == 'invocations' %} + {% elif dataset == "invocations" %} ( command_invocation_id, @@ -49,7 +47,7 @@ dbt_custom_envs ) - {% elif dataset == 'model_executions' %} + {% elif dataset == "model_executions" %} ( command_invocation_id, @@ -62,25 +60,24 @@ query_completed_at, total_node_runtime, rows_affected, - {% if target.type == 'bigquery' %} - bytes_processed, + {% if target.type == "bigquery" %} bytes_processed, {% endif %} materialization, - schema, + "schema", name, alias, message, adapter_response ) - {% elif dataset == 'models' %} + {% elif dataset == "models" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, @@ -93,8 +90,7 @@ all_results ) - - {% elif dataset == 'seed_executions' %} + {% elif dataset == "seed_executions" %} ( command_invocation_id, @@ -108,21 +104,21 @@ total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message, adapter_response ) - {% elif dataset == 'seeds' %} + {% elif dataset == "seeds" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, package_name, path, @@ -132,7 +128,7 @@ all_results ) - {% elif dataset == 'snapshot_executions' %} + {% elif dataset == "snapshot_executions" %} ( command_invocation_id, @@ -146,21 +142,21 @@ total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message, adapter_response ) - {% elif dataset == 'snapshots' %} + {% elif dataset == "snapshots" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, @@ -172,14 +168,14 @@ all_results ) - {% elif dataset == 'sources' %} + {% elif dataset == "sources" %} ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", source_name, loader, name, @@ -189,7 +185,7 @@ all_results ) - {% elif dataset == 'test_executions' %} + {% elif dataset == "test_executions" %} ( command_invocation_id, @@ -207,7 +203,7 @@ adapter_response ) - {% elif dataset == 'tests' %} + {% elif dataset == "tests" %} ( command_invocation_id, @@ -223,8 +219,8 @@ {% else %} - /* No column list available */ - + /* No column list available */ {% endif %} {%- endmacro %} + diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 24f1eb77..8d4afa40 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -59,5 +59,17 @@ {%- endmacro %} +{% macro sqlserver__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} + diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 3e24ae79..2b94f11e 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,108 +1,113 @@ with - base as ( - - select * from {{ ref('stg_dbt__models') }} - - ) - - , model_executions as ( - - select * from {{ ref('stg_dbt__model_executions') }} - - ) - - , latest_models as ( + base as (select * from {{ ref("stg_dbt__models") }}), + model_executions as (select * from {{ ref("stg_dbt__model_executions") }}), + latest_models as ( /* Retrieves the models present in the most recent run */ - select * - from base - where run_started_at = (select max(run_started_at) from base) - - ) + select * from base where run_started_at = (select max(run_started_at) from base) - , latest_models_runs as ( + ), + latest_models_runs as ( /* Retreives all successful run information for the models present in the most recent run and ranks them based on query completion time */ select - model_executions.node_id - , model_executions.was_full_refresh - , model_executions.query_completed_at - , model_executions.total_node_runtime - , model_executions.rows_affected - {% if target.type == 'bigquery' %} - , model_executions.bytes_processed - {% endif %} + model_executions.node_id, + model_executions.was_full_refresh, + model_executions.query_completed_at, + model_executions.total_node_runtime, + model_executions.rows_affected + {% if target.type == "bigquery" %}, model_executions.bytes_processed {% endif %}, /* Row number by refresh and node ID */ - , row_number() over ( + row_number() over ( partition by latest_models.node_id, model_executions.was_full_refresh - order by model_executions.query_completed_at desc /* most recent ranked first */ - ) as run_idx + order by model_executions.query_completed_at desc /* most recent ranked first */ + ) as run_idx, /* Row number by node ID */ - , row_number() over ( - partition by latest_models.node_id - order by model_executions.query_completed_at desc /* most recent ranked first */ + row_number() over ( + partition by latest_models.node_id order by model_executions.query_completed_at desc /* most recent ranked first */ ) as run_idx_id_only from model_executions inner join latest_models on model_executions.node_id = latest_models.node_id where model_executions.status = 'success' - ) - - , latest_model_stats as ( + ), + latest_model_stats as ( select - node_id - , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at - , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime - , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed - {% endif %} - , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at - , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime - , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed - {% endif %} - , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at - , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime - , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed + node_id, + max( + case when was_full_refresh = 1 then query_completed_at end + ) as last_full_refresh_run_completed_at, + max( + case when was_full_refresh = 1 then total_node_runtime end + ) as last_full_refresh_run_total_runtime, + max( + case when was_full_refresh = 1 then rows_affected end + ) as last_full_refresh_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when was_full_refresh then bytes_processed end + ) as last_full_refresh_run_bytes_processed + {% endif %}, + max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at, + max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime + , + max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when run_idx_id_only = 1 then bytes_processed end + ) as last_run_bytes_processed + {% endif %}, + max( + case when not was_full_refresh = 1 then query_completed_at end + ) as last_incremental_run_completed_at, + max( + case when not was_full_refresh = 1 then total_node_runtime end + ) as last_incremental_run_total_runtime, + max( + case when not was_full_refresh = 1 then rows_affected end + ) as last_incremental_run_rows_affected + {% if target.type == "bigquery" %} + , + max( + case when not was_full_refresh then bytes_processed end + ) as last_incremental_run_bytes_processed {% endif %} from latest_models_runs where run_idx = 1 - group by 1 + group by node_id - ) - - , final as ( + ), + final as ( select - latest_models.* - , latest_model_stats.last_full_refresh_run_completed_at - , latest_model_stats.last_full_refresh_run_total_runtime - , latest_model_stats.last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_full_refresh_run_bytes_processed - {% endif %} - , latest_model_stats.last_run_completed_at - , latest_model_stats.last_run_total_runtime - , latest_model_stats.last_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_run_bytes_processed - {% endif %} - , latest_model_stats.last_incremental_run_completed_at - , latest_model_stats.last_incremental_run_total_runtime - , latest_model_stats.last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_incremental_run_bytes_processed + latest_models.*, + latest_model_stats.last_full_refresh_run_completed_at, + latest_model_stats.last_full_refresh_run_total_runtime, + latest_model_stats.last_full_refresh_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_full_refresh_run_bytes_processed + {% endif %}, + latest_model_stats.last_run_completed_at, + latest_model_stats.last_run_total_runtime, + latest_model_stats.last_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_run_bytes_processed + {% endif %}, + latest_model_stats.last_incremental_run_completed_at, + latest_model_stats.last_incremental_run_total_runtime, + latest_model_stats.last_incremental_run_rows_affected + {% if target.type == "bigquery" %} + , latest_model_stats.last_incremental_run_bytes_processed {% endif %} from latest_models - left join latest_model_stats - on latest_models.node_id = latest_model_stats.node_id + left join latest_model_stats on latest_models.node_id = latest_model_stats.node_id ) -select * from final +select * +from final + diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 8ed90ce0..78e6b613 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -1,31 +1,27 @@ with - base as ( - - select * - from {{ ref('stg_dbt__models') }} - - ) - - , models as ( + base as (select * from {{ ref("stg_dbt__models") }}), + models as ( select - model_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , materialization - , tags - , meta - , alias + model_execution_id, + command_invocation_id, + node_id, + run_started_at, + "database", + "schema", + name, + depends_on_nodes, + package_name, + path, + checksum, + materialization, + tags, + meta, + alias from base ) -select * from models +select * +from models + diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index d07c288d..6482394d 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -1,28 +1,24 @@ with - base as ( - - select * - from {{ ref('stg_dbt__seeds') }} - - ) - - , seeds as ( + base as (select * from {{ ref("stg_dbt__seeds") }}), + seeds as ( select - seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , package_name - , path - , checksum - , meta - , alias + seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + "database", + "schema", + name, + package_name, + path, + checksum, + meta, + alias from base ) -select * from seeds +select * +from seeds + diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index a3270714..d84a962a 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -1,30 +1,26 @@ with - base as ( - - select * - from {{ ref('stg_dbt__snapshots') }} - - ) - - , snapshots as ( + base as (select * from {{ ref("stg_dbt__snapshots") }}), + snapshots as ( select - snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , name - , depends_on_nodes - , package_name - , path - , checksum - , strategy - , meta - , alias + snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + "database", + "schema", + name, + depends_on_nodes, + package_name, + path, + checksum, + strategy, + meta, + alias from base ) -select * from snapshots +select * +from snapshots + diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index affe5b99..4cf5f36f 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -1,28 +1,24 @@ with - base as ( - - select * - from {{ ref('stg_dbt__sources') }} - - ) - - , sources as ( + base as (select * from {{ ref("stg_dbt__sources") }}), + sources as ( select - source_execution_id - , command_invocation_id - , node_id - , run_started_at - , database - , schema - , source_name - , loader - , name - , identifier - , loaded_at_field - , freshness + source_execution_id, + command_invocation_id, + node_id, + run_started_at, + "database", + "schema", + source_name, + loader, + name, + identifier, + loaded_at_field, + freshness from base ) -select * from sources +select * +from sources + diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index c6c93548..edc201ee 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -1,35 +1,29 @@ with - base as ( - - select * - from {{ ref('stg_dbt__model_executions') }} - - ) - - , model_executions as ( + base as (select * from {{ ref("stg_dbt__model_executions") }}), + model_executions as ( select - model_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - {% if target.type == 'bigquery' %} - , bytes_processed - {% endif %} - , materialization - , schema -- noqa - , name - , alias - , message + model_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected + {% if target.type == "bigquery" %}, bytes_processed {% endif %}, + materialization, + "schema", -- noqa + name, + alias, + message from base ) -select * from model_executions +select * +from model_executions + diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index 39d2abb5..b9dd605f 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -1,32 +1,28 @@ with - base as ( - - select * - from {{ ref('stg_dbt__seed_executions') }} - - ) - - , seed_executions as ( + base as (select * from {{ ref("stg_dbt__seed_executions") }}), + seed_executions as ( select - seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema - , name - , alias - , message + seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + "schema", + name, + alias, + message from base ) -select * from seed_executions +select * +from seed_executions + diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 91613db0..40774f04 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,32 +1,28 @@ with - base as ( - - select * - from {{ ref('stg_dbt__snapshot_executions') }} - - ) - - , snapshot_executions as ( + base as (select * from {{ ref("stg_dbt__snapshot_executions") }}), + snapshot_executions as ( select - snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema - , name - , alias - , message + snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + "schema", + name, + alias, + message from base ) -select * from snapshot_executions +select * +from snapshot_executions + diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 14d6c6b2..193835ae 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -21,7 +21,7 @@ select , cast(null as {{ type_int() }}) as bytes_processed {% endif %} , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_string() }}) as alias , cast(null as {{ type_string() }}) as message diff --git a/models/sources/models.sql b/models/sources/models.sql index debeaf05..d58bb4dd 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -10,8 +10,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id , cast(null as {{ type_string() }}) as node_id , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "database" + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_array() }}) as depends_on_nodes , cast(null as {{ type_string() }}) as package_name diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index c3e9465d..2ca0a656 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -18,7 +18,7 @@ select , cast(null as {{ type_float() }}) as total_node_runtime , cast(null as {{ type_int() }}) as rows_affected , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_string() }}) as alias , cast(null as {{ type_string() }}) as message diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 3c99ef01..5a7b0162 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -10,8 +10,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id , cast(null as {{ type_string() }}) as node_id , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "database" + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_string() }}) as package_name , cast(null as {{ type_string() }}) as path diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index c3e9465d..2ca0a656 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -18,7 +18,7 @@ select , cast(null as {{ type_float() }}) as total_node_runtime , cast(null as {{ type_int() }}) as rows_affected , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_string() }}) as alias , cast(null as {{ type_string() }}) as message diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index c47ef812..dd038939 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -9,8 +9,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id , cast(null as {{ type_string() }}) as node_id , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "database" + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as name , cast(null as {{ type_array() }}) as depends_on_nodes , cast(null as {{ type_string() }}) as package_name diff --git a/models/sources/sources.sql b/models/sources/sources.sql index d152ad22..5a9f5be9 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -8,8 +8,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id , cast(null as {{ type_string() }}) as node_id , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as database - , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as "database" + , cast(null as {{ type_string() }}) as "schema" , cast(null as {{ type_string() }}) as source_name , cast(null as {{ type_string() }}) as loader , cast(null as {{ type_string() }}) as name diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 1381f6c4..1e1ad57b 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -24,7 +24,7 @@ with , bytes_processed {% endif %} , materialization - , schema -- noqa + , "schema" -- noqa , name , alias , message diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index 0af3b491..eb5fc5c6 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -13,8 +13,8 @@ with , command_invocation_id , node_id , run_started_at - , database - , schema + , "database" + , "schema" , name , depends_on_nodes , package_name diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index ec90aaf0..31b0c2e9 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -21,7 +21,7 @@ with , total_node_runtime , rows_affected , materialization - , schema -- noqa + , "schema" -- noqa , name , alias , message diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 6e652631..5aea6e08 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -13,8 +13,8 @@ with , command_invocation_id , node_id , run_started_at - , database - , schema + , "database" + , "schema" , name , package_name , path diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 5818133d..dbac0734 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -21,7 +21,7 @@ with , total_node_runtime , rows_affected , materialization - , schema -- noqa + , "schema" -- noqa , name , alias , message diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index 7571e14b..ffdd0732 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -13,8 +13,8 @@ with , command_invocation_id , node_id , run_started_at - , database - , schema + , "database" + , "schema" , name , depends_on_nodes , package_name diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index aca60a43..5c4056fc 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -13,8 +13,8 @@ with , command_invocation_id , node_id , run_started_at - , database - , schema + , "database" + , "schema" , source_name , loader , name From 25679729c4e113bb63dfadbce331df8bf874bcef Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 11:15:11 +0200 Subject: [PATCH 08/18] Added dbt-sqlserver 1.8.4 support --- tox.ini | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 480d7821..50e79ce7 100644 --- a/tox.ini +++ b/tox.ini @@ -355,7 +355,7 @@ commands = [testenv:integration_sqlserver] changedir = integration_test_project -deps = dbt-sqlserver~=1.7.4 +deps = dbt-sqlserver~=1.8.4 commands = dbt clean dbt deps @@ -376,3 +376,11 @@ commands = dbt clean dbt deps dbt build --target sqlserver + +[testenv:integration_sqlserver_1_8_4] +changedir = integration_test_project +deps = dbt-sqlserver~=1.8.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver From 479ba5994d71df1c142cbc472d850694aa6d6bf9 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 18 Sep 2024 14:48:23 +0200 Subject: [PATCH 09/18] Added CI for SQL Server --- .github/workflows/ci_test_package.yml | 8 +++++++- .github/workflows/main_test_package.yml | 8 +++++++- integration_test_project/profiles.yml | 2 +- tox.ini | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 06c07e92..5eb359fd 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -113,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest @@ -145,6 +145,12 @@ jobs: - name: Install tox run: python3 -m pip install tox + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + - name: Checkout uses: actions/checkout@v3 with: diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 12a9c957..1c55194e 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres"] + warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest permissions: @@ -66,6 +66,12 @@ jobs: - name: Install tox run: python3 -m pip install tox + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + - id: auth if: ${{ matrix.warehouse == 'bigquery' }} uses: google-github-actions/auth@v1 diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index c5188699..33a1117a 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -61,5 +61,5 @@ dbt_artifacts: schema: dbo windows_login: False trust_cert: True - user: dbt + user: sa password: "123" diff --git a/tox.ini b/tox.ini index 50e79ce7..410168bc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ skipsdist = True envlist = lint_all [sqlfluff] -dialect = snowflake +dialect = tsql templater = dbt rules = LT01,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP03,ST01,LT09,AM03,CP04,LT10,ST05,ST03,JJ01,AM05,CV08 @@ -35,7 +35,7 @@ rules = LT01,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP deps = sqlfluff-templater-dbt~=2.0.2 - dbt-snowflake~=1.8.0 + dbt-sqlserver~=1.8.4 [sqlfluff:indentation] indent_unit = space From 1f8100e0df1f9712399779d66fe98c25e5315acb Mon Sep 17 00:00:00 2001 From: KijkEr Date: Thu, 19 Sep 2024 12:08:11 +0200 Subject: [PATCH 10/18] Changed target.name to target.type for sqlserver persist_docs columns --- dbt_project.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7d296b51..492a9e0b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -15,9 +15,8 @@ models: +file_format: delta sources: +materialized: incremental - +on_schema_change: append_new_columns +full_refresh: false +persist_docs: # Databricks and SQL Server don't offer column-level support for persisting docs - columns: '{{ target.name != "databricks" and target.name != "sqlserver" }}' + columns: '{{ target.name != "databricks" and target.type != "sqlserver" }}' +as_columnstore: False From d07ce3375ebfe1f5350bf78db73b7b15639fa011 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Thu, 19 Sep 2024 12:08:51 +0200 Subject: [PATCH 11/18] Added persist_docs relation exclusion for sqlserver --- dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_project.yml b/dbt_project.yml index 492a9e0b..4c05e3ae 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,4 +19,5 @@ models: +persist_docs: # Databricks and SQL Server don't offer column-level support for persisting docs columns: '{{ target.name != "databricks" and target.type != "sqlserver" }}' + relation: '{{ target.type != "sqlserver" }}' +as_columnstore: False From 7110f297c9304888d6025701c6471c6c23497724 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 07:43:29 +0200 Subject: [PATCH 12/18] Set delimited idenitifier conditional for database and schema columns --- .../upload_results/get_column_name_lists.sql | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index 37f22500..707e0c2c 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -63,7 +63,9 @@ {% if target.type == "bigquery" %} bytes_processed, {% endif %} materialization, - "schema", + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, @@ -76,8 +78,9 @@ command_invocation_id, node_id, run_started_at, - "database", - "schema", + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, depends_on_nodes, package_name, @@ -104,7 +107,9 @@ total_node_runtime, rows_affected, materialization, - "schema", + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, @@ -117,8 +122,9 @@ command_invocation_id, node_id, run_started_at, - "database", - "schema", + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, package_name, path, @@ -142,7 +148,9 @@ total_node_runtime, rows_affected, materialization, - "schema", + {% if target.type == "sqlserver" %} "schema", + {% else %} schema, + {% endif %} name, alias, message, @@ -155,8 +163,9 @@ command_invocation_id, node_id, run_started_at, - "database", - "schema", + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} name, depends_on_nodes, package_name, @@ -174,8 +183,9 @@ command_invocation_id, node_id, run_started_at, - "database", - "schema", + {% if target.type == "sqlserver" %} "database", "schema", + {% else %} database, schema, + {% endif %} source_name, loader, name, From bc97fc08a16f71e93b2ada6febaa3439430432d1 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 07:49:58 +0200 Subject: [PATCH 13/18] Changed tox.ini back to original --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 410168bc..50e79ce7 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ skipsdist = True envlist = lint_all [sqlfluff] -dialect = tsql +dialect = snowflake templater = dbt rules = LT01,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP03,ST01,LT09,AM03,CP04,LT10,ST05,ST03,JJ01,AM05,CV08 @@ -35,7 +35,7 @@ rules = LT01,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP deps = sqlfluff-templater-dbt~=2.0.2 - dbt-sqlserver~=1.8.4 + dbt-snowflake~=1.8.0 [sqlfluff:indentation] indent_unit = space From 114b87cbc275b7081c9b53cae6eaa4c560a69321 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 07:55:43 +0200 Subject: [PATCH 14/18] Added delimitied identifier conditional for sql server --- models/sources/model_executions.sql | 48 ++++++++++++-------------- models/sources/models.sql | 42 +++++++++++----------- models/sources/seed_executions.sql | 42 +++++++++++----------- models/sources/snapshot_executions.sql | 42 +++++++++++----------- models/sources/snapshots.sql | 39 +++++++++++---------- models/sources/sources.sql | 40 ++++++++++----------- 6 files changed, 124 insertions(+), 129 deletions(-) diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 193835ae..fb76e983 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -1,30 +1,28 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - {% if target.type == 'bigquery' %} - , cast(null as {{ type_int() }}) as bytes_processed - {% endif %} - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected + {% if target.type == "bigquery" %} + , cast(null as {{ type_int() }}) as bytes_processed + {% endif %}, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/models.sql b/models/sources/models.sql index d58bb4dd..7ac8ee84 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -1,26 +1,26 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as "database" - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_array() }}) as depends_on_nodes - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_array() }}) as tags - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_array() }}) as tags, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index 2ca0a656..95a750e6 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -1,27 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index 2ca0a656..95a750e6 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -1,27 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_boolean() }}) as was_full_refresh - , cast(null as {{ type_string() }}) as thread_id - , cast(null as {{ type_string() }}) as status - , cast(null as {{ type_timestamp() }}) as compile_started_at - , cast(null as {{ type_timestamp() }}) as query_completed_at - , cast(null as {{ type_float() }}) as total_node_runtime - , cast(null as {{ type_int() }}) as rows_affected - , cast(null as {{ type_string() }}) as materialization - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_string() }}) as message, + cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 + diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index dd038939..f4992be4 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -1,24 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as "database" - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_array() }}) as depends_on_nodes - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_string() }}) as strategy - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %},, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as strategy, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + diff --git a/models/sources/sources.sql b/models/sources/sources.sql index 5a9f5be9..a6a1ee85 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -1,25 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - select 1 as foo - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as "database" - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as source_name - , cast(null as {{ type_string() }}) as loader - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as identifier - , cast(null as {{ type_string() }}) as loaded_at_field - {% if target.type == 'snowflake' %} - , cast(null as {{ type_array() }}) as freshness - {% else %} - , cast(null as {{ type_json() }}) as freshness - {% endif %} - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %},, + cast(null as {{ type_string() }}) as source_name, + cast(null as {{ type_string() }}) as loader, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as identifier, + cast(null as {{ type_string() }}) as loaded_at_field + {% if target.type == "snowflake" %}, cast(null as {{ type_array() }}) as freshness + {% else %}, cast(null as {{ type_json() }}) as freshness + {% endif %}, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + From 14a923937dcefff305eeb1b861552ed99bb2fbd8 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 08:44:48 +0200 Subject: [PATCH 15/18] Added delimited identifier conditional for sql server --- models/dim_dbt__models.sql | 8 ++- models/dim_dbt__seeds.sql | 8 ++- models/dim_dbt__snapshots.sql | 8 ++- models/dim_dbt__sources.sql | 8 ++- models/fct_dbt__model_executions.sql | 4 +- models/fct_dbt__seed_executions.sql | 4 +- models/fct_dbt__snapshot_executions.sql | 4 +- models/fct_dbt__test_executions.sql | 40 ++++++-------- models/sources/model_executions.sql | 2 +- models/sources/snapshots.sql | 4 +- models/staging/stg_dbt__model_executions.sql | 55 +++++++++---------- models/staging/stg_dbt__models.sql | 49 +++++++++-------- models/staging/stg_dbt__seed_executions.sql | 51 +++++++++-------- models/staging/stg_dbt__seeds.sql | 43 ++++++++------- .../staging/stg_dbt__snapshot_executions.sql | 51 +++++++++-------- models/staging/stg_dbt__snapshots.sql | 46 ++++++++-------- models/staging/stg_dbt__sources.sql | 43 ++++++++------- 17 files changed, 223 insertions(+), 205 deletions(-) diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 78e6b613..0836b0e9 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -7,9 +7,13 @@ with command_invocation_id, node_id, run_started_at, - "database", - "schema", name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, depends_on_nodes, package_name, path, diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index 6482394d..02b16dff 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -7,9 +7,13 @@ with command_invocation_id, node_id, run_started_at, - "database", - "schema", name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, package_name, path, checksum, diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index d84a962a..43d7c004 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -7,9 +7,13 @@ with command_invocation_id, node_id, run_started_at, - "database", - "schema", name, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, depends_on_nodes, package_name, path, diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index 4cf5f36f..f8a53552 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -7,8 +7,12 @@ with command_invocation_id, node_id, run_started_at, - "database", - "schema", + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, source_name, loader, name, diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index edc201ee..43a43792 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -16,7 +16,9 @@ with rows_affected {% if target.type == "bigquery" %}, bytes_processed {% endif %}, materialization, - "schema", -- noqa + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, name, alias, message diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index b9dd605f..f537c9da 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -15,7 +15,9 @@ with total_node_runtime, rows_affected, materialization, - "schema", + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, name, alias, message diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 40774f04..ee35c616 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -15,7 +15,9 @@ with total_node_runtime, rows_affected, materialization, - "schema", + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, name, alias, message diff --git a/models/fct_dbt__test_executions.sql b/models/fct_dbt__test_executions.sql index 43b87d33..9a4727e1 100644 --- a/models/fct_dbt__test_executions.sql +++ b/models/fct_dbt__test_executions.sql @@ -1,29 +1,25 @@ with - base as ( - - select * - from {{ ref('stg_dbt__test_executions') }} - - ) - - , test_executions as ( + base as (select * from {{ ref("stg_dbt__test_executions") }}), + test_executions as ( select - test_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , failures - , message + test_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + failures, + message from base ) -select * from test_executions +select * +from test_executions + diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index fb76e983..e07107cf 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -11,7 +11,7 @@ select cast(null as {{ type_timestamp() }}) as compile_started_at, cast(null as {{ type_timestamp() }}) as query_completed_at, cast(null as {{ type_float() }}) as total_node_runtime, - cast(null as {{ type_int() }}) as rows_affected + cast(null as {{ type_int() }}) as rows_affected, {% if target.type == "bigquery" %} , cast(null as {{ type_int() }}) as bytes_processed {% endif %}, diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index f4992be4..77061731 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -4,13 +4,13 @@ with dummy_cte as (select 1 as foo) select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at + cast(null as {{ type_timestamp() }}) as run_started_at, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" {% else %} database {% endif %}, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" {% else %} schema - {% endif %},, + {% endif %}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_array() }}) as depends_on_nodes, cast(null as {{ type_string() }}) as package_name, diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 1e1ad57b..ad2c4f50 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,36 +1,33 @@ with - base as ( - - select * - from {{ ref('model_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("model_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - {% if target.type == 'bigquery' %} - , bytes_processed - {% endif %} - , materialization - , "schema" -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as model_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected + {% if target.type == "bigquery" %}, bytes_processed {% endif %}, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index eb5fc5c6..d8184183 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,31 +1,32 @@ with - base as ( - - select * - from {{ ref('models') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("models") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id - , command_invocation_id - , node_id - , run_started_at - , "database" - , "schema" - , name - , depends_on_nodes - , package_name - , path - , checksum - , materialization - , tags - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as model_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + depends_on_nodes, + package_name, + path, + checksum, + materialization, + tags, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 31b0c2e9..90f52870 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,33 +1,32 @@ with - base as ( - - select * - from {{ ref('seed_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("seed_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , "schema" -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 5aea6e08..0e5dcb1d 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,28 +1,29 @@ with - base as ( - - select * - from {{ ref('seeds') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("seeds") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id - , command_invocation_id - , node_id - , run_started_at - , "database" - , "schema" - , name - , package_name - , path - , checksum - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as seed_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + package_name, + path, + checksum, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index dbac0734..8794b99b 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,33 +1,32 @@ with - base as ( - - select * - from {{ ref('snapshot_executions') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("snapshot_executions") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , "schema" -- noqa - , name - , alias - , message - , adapter_response + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + {{ split_part("thread_id", "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias, + message, + adapter_response from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index ffdd0732..8b2a66a4 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -1,30 +1,32 @@ with - base as ( + base as (select * from {{ ref("snapshots") }}), - select * - from {{ ref('snapshots') }} - - ) - - , enhanced as ( + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , "database" - , "schema" - , name - , depends_on_nodes - , package_name - , path - , checksum - , strategy - , meta - , alias + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + name, + depends_on_nodes, + package_name, + path, + checksum, + strategy, + meta, + alias from base ) -select * from enhanced +select * +from enhanced + diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index 5c4056fc..1bd8cef1 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,28 +1,29 @@ with - base as ( - - select * - from {{ ref('sources') }} - - ) - - , enhanced as ( + base as (select * from {{ ref("sources") }}), + enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id - , command_invocation_id - , node_id - , run_started_at - , "database" - , "schema" - , source_name - , loader - , name - , identifier - , loaded_at_field - , freshness + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "node_id"]) }} + as source_execution_id, + command_invocation_id, + node_id, + run_started_at, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + source_name, + loader, + name, + identifier, + loaded_at_field, + freshness from base ) -select * from enhanced +select * +from enhanced + From d23425973892dbfb32f297012d0263a1c473a41b Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 09:50:39 +0200 Subject: [PATCH 16/18] Fixed postgres issues --- models/dim_dbt__current_models.sql | 35 ++++++++++++++++++++++------- models/sources/model_executions.sql | 2 +- models/sources/sources.sql | 4 ++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 2b94f11e..778385a0 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -37,13 +37,22 @@ with select node_id, max( - case when was_full_refresh = 1 then query_completed_at end + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then query_completed_at + end ) as last_full_refresh_run_completed_at, max( - case when was_full_refresh = 1 then total_node_runtime end + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then total_node_runtime + end ) as last_full_refresh_run_total_runtime, max( - case when was_full_refresh = 1 then rows_affected end + case + when was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then rows_affected + end ) as last_full_refresh_run_rows_affected {% if target.type == "bigquery" %} , @@ -52,8 +61,9 @@ with ) as last_full_refresh_run_bytes_processed {% endif %}, max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at, - max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime - , + max( + case when run_idx_id_only = 1 then total_node_runtime end + ) as last_run_total_runtime, max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected {% if target.type == "bigquery" %} , @@ -62,13 +72,22 @@ with ) as last_run_bytes_processed {% endif %}, max( - case when not was_full_refresh = 1 then query_completed_at end + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then query_completed_at + end ) as last_incremental_run_completed_at, max( - case when not was_full_refresh = 1 then total_node_runtime end + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then total_node_runtime + end ) as last_incremental_run_total_runtime, max( - case when not was_full_refresh = 1 then rows_affected end + case + when not was_full_refresh {% if target.type == "sqlserver" %} = 1 {% endif %} + then rows_affected + end ) as last_incremental_run_rows_affected {% if target.type == "bigquery" %} , diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index e07107cf..fa57379f 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -14,7 +14,7 @@ select cast(null as {{ type_int() }}) as rows_affected, {% if target.type == "bigquery" %} , cast(null as {{ type_int() }}) as bytes_processed - {% endif %}, + {% endif %} cast(null as {{ type_string() }}) as materialization, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" {% else %} schema diff --git a/models/sources/sources.sql b/models/sources/sources.sql index a6a1ee85..91c8d1d3 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -4,13 +4,13 @@ with dummy_cte as (select 1 as foo) select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at + cast(null as {{ type_timestamp() }}) as run_started_at, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" {% else %} database {% endif %}, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" {% else %} schema - {% endif %},, + {% endif %}, cast(null as {{ type_string() }}) as source_name, cast(null as {{ type_string() }}) as loader, cast(null as {{ type_string() }}) as name, From 06bdd72f0a09fb5c236ae3d51c385acb77dc4ba0 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 10:01:44 +0200 Subject: [PATCH 17/18] Fix linting errors --- models/dim_dbt__current_models.sql | 3 +++ models/dim_dbt__models.sql | 1 + models/dim_dbt__seeds.sql | 1 + models/dim_dbt__snapshots.sql | 1 + models/dim_dbt__sources.sql | 1 + models/fct_dbt__model_executions.sql | 1 + models/fct_dbt__seed_executions.sql | 1 + models/fct_dbt__snapshot_executions.sql | 1 + models/fct_dbt__test_executions.sql | 1 + 9 files changed, 11 insertions(+) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 778385a0..4e6ceb1b 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -7,6 +7,7 @@ with select * from base where run_started_at = (select max(run_started_at) from base) ), + latest_models_runs as ( /* Retreives all successful run information for the models present in the most @@ -32,6 +33,7 @@ with where model_executions.status = 'success' ), + latest_model_stats as ( select @@ -100,6 +102,7 @@ with group by node_id ), + final as ( select diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 0836b0e9..34f066a2 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__models") }}), + models as ( select diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index 02b16dff..cf005fbf 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__seeds") }}), + seeds as ( select diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 43d7c004..d1d0a999 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__snapshots") }}), + snapshots as ( select diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index f8a53552..c6e3263a 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__sources") }}), + sources as ( select diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index 43a43792..3b7c2368 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__model_executions") }}), + model_executions as ( select diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index f537c9da..1e6465a0 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__seed_executions") }}), + seed_executions as ( select diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index ee35c616..58ca4408 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__snapshot_executions") }}), + snapshot_executions as ( select diff --git a/models/fct_dbt__test_executions.sql b/models/fct_dbt__test_executions.sql index 9a4727e1..c9e347f0 100644 --- a/models/fct_dbt__test_executions.sql +++ b/models/fct_dbt__test_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("stg_dbt__test_executions") }}), + test_executions as ( select From d189c98b20508a230c2d2c0bcf8adce688f1ee1e Mon Sep 17 00:00:00 2001 From: KijkEr Date: Fri, 20 Sep 2024 13:45:42 +0200 Subject: [PATCH 18/18] More fixes for Bigquery --- models/sources/model_executions.sql | 2 +- models/sources/seeds.sql | 36 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index fa57379f..5710affd 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -13,7 +13,7 @@ select cast(null as {{ type_float() }}) as total_node_runtime, cast(null as {{ type_int() }}) as rows_affected, {% if target.type == "bigquery" %} - , cast(null as {{ type_int() }}) as bytes_processed + cast(null as {{ type_int() }}) as bytes_processed, {% endif %} cast(null as {{ type_string() }}) as materialization, cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 5a7b0162..882c661a 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -1,23 +1,23 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with - dummy_cte as ( - - select 1 as foo - - ) +with dummy_cte as (select 1 as foo) select - cast(null as {{ type_string() }}) as command_invocation_id - , cast(null as {{ type_string() }}) as node_id - , cast(null as {{ type_timestamp() }}) as run_started_at - , cast(null as {{ type_string() }}) as "database" - , cast(null as {{ type_string() }}) as "schema" - , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_string() }}) as path - , cast(null as {{ type_string() }}) as checksum - , cast(null as {{ type_json() }}) as meta - , cast(null as {{ type_string() }}) as alias - , cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, + cast(null as {{ type_string() }}) as {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_json() }}) as meta, + cast(null as {{ type_string() }}) as alias, + cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 +