Skip to content

Commit

Permalink
Update to include new features
Browse files Browse the repository at this point in the history
  • Loading branch information
Gemma Down committed Sep 29, 2023
1 parent 5f25576 commit a292b45
Show file tree
Hide file tree
Showing 12 changed files with 107 additions and 301 deletions.
25 changes: 5 additions & 20 deletions macros/upload_individual_datasets/upload_exposures.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,23 +88,6 @@
{% macro postgres__get_exposures_dml_sql(exposures) -%}
{% if exposures != [] %}

{% set columns = [
'command_invocation_id',
'node_id',
'run_started_at',
'name',
'type',
'owner',
'maturity',
'path',
'description',
'url',
'package_name',
'depends_on_nodes',
'tags',
'all_results',
] %}

{% set exposure_values %}
{% for exposure in exposures -%}
(
Expand All @@ -121,13 +104,15 @@
'{{ exposure.package_name }}', {# package_name #}
$${{ tojson(exposure.depends_on.nodes) }}$$, {# depends_on_nodes #}
$${{ tojson(exposure.tags) }}$$, {# tags #}
$${{ tojson(exposure) }}$$ {# all_results #}
{% if var('dbt_artifacts_exclude_all_results', false) %}
null
{% else %}
$${{ tojson(exposure) }}$$ {# all_results #}
{% endif %}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}
{{ "(" ~ columns | join(', ') ~ ")"}}
VALUES
{{ exposure_values }}
{% else %}
{{ return("") }}
Expand Down
121 changes: 48 additions & 73 deletions macros/upload_individual_datasets/upload_invocations.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,88 +161,63 @@
{% endmacro -%}

{% macro postgres__get_invocations_dml_sql() -%}
{% set columns = [
'command_invocation_id',
'dbt_version',
'project_name',
'run_started_at',
'dbt_command',
'full_refresh_flag',
'target_profile_name',
'target_name',
'target_schema',
'target_threads',
'dbt_cloud_project_id',
'dbt_cloud_job_id',
'dbt_cloud_run_id',
'dbt_cloud_run_reason_category',
'dbt_cloud_run_reason',
'env_vars',
'dbt_vars',
'invocation_args',
'dbt_custom_envs',
] %}

{% set invocation_values %}
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ dbt_version }}', {# dbt_version #}
'{{ project_name }}', {# project_name #}
'{{ run_started_at }}', {# run_started_at #}
'{{ flags.WHICH }}', {# dbt_command #}
{{ flags.FULL_REFRESH }}, {# full_refresh_flag #}
'{{ target.profile_name }}', {# target_profile_name #}
'{{ target.name }}', {# target_name #}
'{{ target.schema }}', {# target_schema #}
{{ target.threads }}, {# target_threads #}

'{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #}
'{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #}
'{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #}
'{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #}
$${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #}

{% if var('env_vars', none) %}
{% set env_vars_dict = {} %}
{% for env_variable in var('env_vars') %}
{% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %}
{% endfor %}
$${{ tojson(env_vars_dict) }}$$, {# env_vars #}
{% else %}
null, {# env_vars #}
{% endif %}

{% if var('dbt_vars', none) %}
{% set dbt_vars_dict = {} %}
{% for dbt_var in var('dbt_vars') %}
{% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %}
{% endfor %}
$${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #}
{% else %}
null, {# dbt_vars #}
{% endif %}
'{{ invocation_id }}', {# command_invocation_id #}
'{{ dbt_version }}', {# dbt_version #}
'{{ project_name }}', {# project_name #}
'{{ run_started_at }}', {# run_started_at #}
'{{ flags.WHICH }}', {# dbt_command #}
{{ flags.FULL_REFRESH }}, {# full_refresh_flag #}
'{{ target.profile_name }}', {# target_profile_name #}
'{{ target.name }}', {# target_name #}
'{{ target.schema }}', {# target_schema #}
{{ target.threads }}, {# target_threads #}

'{{ env_var("DBT_CLOUD_PROJECT_ID", "") }}', {# dbt_cloud_project_id #}
'{{ env_var("DBT_CLOUD_JOB_ID", "") }}', {# dbt_cloud_job_id #}
'{{ env_var("DBT_CLOUD_RUN_ID", "") }}', {# dbt_cloud_run_id #}
'{{ env_var("DBT_CLOUD_RUN_REASON_CATEGORY", "") }}', {# dbt_cloud_run_reason_category #}
$${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #}

{% if var('env_vars', none) %}
{% set env_vars_dict = {} %}
{% for env_variable in var('env_vars') %}
{% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %}
{% endfor %}
$${{ tojson(env_vars_dict) }}$$, {# env_vars #}
{% else %}
null, {# env_vars #}
{% endif %}

{% if invocation_args_dict.vars %}
{# vars - different format for pre v1.5 (yaml vs list) #}
{% if invocation_args_dict.vars is string %}
{# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #}
{% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %}
{% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %}
{% if var('dbt_vars', none) %}
{% set dbt_vars_dict = {} %}
{% for dbt_var in var('dbt_vars') %}
{% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %}
{% endfor %}
$${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #}
{% else %}
null, {# dbt_vars #}
{% endif %}
{% endif %}

$${{ tojson(invocation_args_dict) }}$$, {# invocation_args #}
{% if invocation_args_dict.vars %}
{# vars - different format for pre v1.5 (yaml vs list) #}
{% if invocation_args_dict.vars is string %}
{# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #}
{% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %}
{% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %}
{% endif %}
{% endif %}

{% set metadata_env = {} %}
{% for key, value in dbt_metadata_envs.items() %}
{% do metadata_env.update({key: value}) %}
{% endfor %}
$${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #}
$${{ tojson(invocation_args_dict) }}$$, {# invocation_args #}

{% set metadata_env = {} %}
{% for key, value in dbt_metadata_envs.items() %}
{% do metadata_env.update({key: value}) %}
{% endfor %}
$${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #}
)
{% endset %}
{{ "(" ~ columns | join(', ') + ")" }}
VALUES
{{ invocation_values }}

{% endmacro -%}
80 changes: 22 additions & 58 deletions macros/upload_individual_datasets/upload_model_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,75 +165,39 @@

{% macro postgres__get_model_executions_dml_sql(models) -%}
{% if models != [] %}
{% set columns = [
'command_invocation_id',
'node_id',
'run_started_at',
'was_full_refresh',
'thread_id',
'status',
'compile_started_at',
'query_completed_at',
'total_node_runtime',
'rows_affected',
'materialization',
'schema',
'name',
'alias',
'message',
'adapter_response',
]%}
{% set model_execution_values %}
{% for model in models -%}
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ model.node.unique_id }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}
'{{ invocation_id }}', {# command_invocation_id #}
'{{ model.node.unique_id }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}

{% set config_full_refresh = model.node.config.full_refresh %}
{% if config_full_refresh is none %}
{% set config_full_refresh = flags.FULL_REFRESH %}
{% endif %}
{{ config_full_refresh }}, {# was_full_refresh #}
{% set config_full_refresh = model.node.config.full_refresh %}
{% if config_full_refresh is none %}
{% set config_full_refresh = flags.FULL_REFRESH %}
{% endif %}
{{ config_full_refresh }}, {# was_full_refresh #}

'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null,
'{{ model.node.config.materialized }}', {# materialization #}
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
$${{ model.message }}$$, {# message #}
$${{ tojson(model.adapter_response) }}$$ {# adapter_response #}
{{ model.execution_time }}, {# total_node_runtime #}
null, {# rows_affected #}
'{{ model.node.config.materialized }}', {# materialization #}
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
$${{ model.message }}$$, {# message #}
$${{ tojson(model.adapter_response) }}$$ {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}
{{ "(" ~ columns | join(', ') ~ ")"}}
VALUES
{{ model_execution_values }}
{% else %}
{{ return("") }}
Expand Down
25 changes: 5 additions & 20 deletions macros/upload_individual_datasets/upload_models.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,23 +92,6 @@

{% macro postgres__get_models_dml_sql(models) -%}
{% if models != [] %}
{% set columns = [
'command_invocation_id',
'node_id',
'run_started_at',
'database',
'schema',
'name',
'depends_on_nodes',
'package_name',
'path',
'checksum',
'materialization',
'tags',
'meta',
'alias',
'all_results',
] %}
{% set model_values %}
{% for model in models -%}
{% do model.pop('raw_code', None) %}
Expand All @@ -127,13 +110,15 @@
'{{ tojson(model.tags) }}', {# tags #}
$${{ model.config.meta }}$$, {# meta #}
'{{ model.alias }}', {# alias #}
$${{ tojson(model) }}$$ {# all_results #}
{% if var('dbt_artifacts_exclude_all_results', false) %}
null
{% else %}
$${{ tojson(model) }}$$ {# all_results #}
{% endif %}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}
{{ "(" ~ columns | join(', ') ~ ")"}}
VALUES
{{ model_values }}
{% else %}
{{ return("") }}
Expand Down
20 changes: 0 additions & 20 deletions macros/upload_individual_datasets/upload_seed_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,24 +211,6 @@

{% macro postgres__get_seed_executions_dml_sql(seeds) -%}
{% if seeds != [] %}
{% set columns = [
'command_invocation_id',
'node_id',
'run_started_at',
'was_full_refresh',
'thread_id',
'status',
'compile_started_at',
'query_completed_at',
'total_node_runtime',
'rows_affected',
'materialization',
'schema',
'name',
'alias',
'message',
'adapter_response',
]%}
{% set seed_execution_values %}
{% for model in seeds -%}
(
Expand Down Expand Up @@ -278,8 +260,6 @@
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}
{{ "(" ~ columns | join(', ') ~ ")"}}
VALUES
{{ seed_execution_values }}
{% else %}
{{ return("") }}
Expand Down
22 changes: 5 additions & 17 deletions macros/upload_individual_datasets/upload_seeds.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,20 +81,6 @@

{% macro postgres__get_seeds_dml_sql(seeds) -%}
{% if seeds != [] %}
{% set columns = [
'command_invocation_id',
'node_id',
'run_started_at',
'database',
'schema',
'name',
'package_name',
'path',
'checksum',
'meta',
'alias',
'all_results',
] %}
{% set seed_values %}
{% for seed in seeds -%}
(
Expand All @@ -109,13 +95,15 @@
'{{ seed.checksum.checksum }}', {# checksum #}
$${{ tojson(seed.config.meta) }}$$, {# meta #}
'{{ seed.alias }}', {# alias #}
$${{ tojson(seed) }}$$ {# all_results #}
{% if var('dbt_artifacts_exclude_all_results', false) %}
null
{% else %}
$${{ tojson(seed) }}$$ {# all_results #}
{% endif %}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}
{{ "(" ~ columns | join(', ') ~ ")"}}
VALUES
{{ seed_values }}
{% else %}
{{ return("") }}
Expand Down
Loading

0 comments on commit a292b45

Please sign in to comment.