From 8046e29314ca4a0d23338a8c8b701d327270cbee Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 11 Jun 2020 19:45:28 -0400 Subject: [PATCH 1/2] Reference static dics as macros --- macros/hooks/model_audit.sql | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/macros/hooks/model_audit.sql b/macros/hooks/model_audit.sql index d728a30..b49bcde 100644 --- a/macros/hooks/model_audit.sql +++ b/macros/hooks/model_audit.sql @@ -1,13 +1,19 @@ -{% set _audit_table_columns = { +{% macro _audit_table_columns() %} + +{% do return ({ 'model': 'string', 'schema': 'string', 'created_at': dbt_utils.type_timestamp(), 'training_info': 'array>>>', 'feature_info': 'array>', 'weights': 'array>>>', -} %} +}) %} + +{% endmacro %} -{% set _audit_insert_templates = { +{% macro _audit_insert_templates() %} + +{% do return ({ 'default': { 'training_info': [ 'training_run', @@ -33,19 +39,21 @@ ], 'feature_info': ['*'] } -} %} +}) %} + +{% endmacro %} {% macro _get_audit_info_cols(model_type, info_type) %} {% set cols = none %} - {% if model_type in _audit_insert_templates.keys() %} - {% if info_type in _audit_insert_templates[model_type].keys() %} - {% set cols = _audit_insert_templates[model_type][info_type] %} + {% if model_type in dbt_ml._audit_insert_templates().keys() %} + {% if info_type in dbt_ml._audit_insert_templates()[model_type].keys() %} + {% set cols = dbt_ml._audit_insert_templates()[model_type][info_type] %} {% endif %} {% endif %} {% if cols is none %} - {% set cols = _audit_insert_templates['default'][info_type] %} + {% set cols = dbt_ml._audit_insert_templates()['default'][info_type] %} {% endif %} {% do return(cols) %} @@ -55,7 +63,7 @@ {% macro model_audit() %} {% set model_type = config.get('ml_config')['model_type'] %} - {% set model_type_repr = _audit_insert_templates.get(model_type, 'default') %} + {% set model_type_repr = dbt_ml._audit_insert_templates().get(model_type, 'default') %} {% set info_types = ['training_info', 'feature_info', 'weights'] %} @@ -68,8 +76,8 @@ current_timestamp as created_at, {% for info_type in info_types %} - {% if info_type not in _audit_insert_templates[model_type_repr] %} - cast(null as {{ _audit_table_columns[info_type] }}) as {{ info_type }} + {% if info_type not in dbt_ml._audit_insert_templates()[model_type_repr] %} + cast(null as {{ dbt_ml._audit_table_columns()[info_type] }}) as {{ info_type }} {% else %} array( select as struct {{ dbt_ml._get_audit_info_cols(model_type, info_type) | join(', ') }} @@ -95,7 +103,7 @@ {% if not audit_table_exists -%} create table if not exists {{ audit_table }} ( - {% for column, type in _audit_table_columns.items() %} + {% for column, type in dbt_ml._audit_table_columns().items() %} {{ column }} {{ type }}{% if not loop.last %},{% endif %} {% endfor %} ) From b0e2555279cd9a78ab9f9f395bc911ba95a3df05 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 11 Jun 2020 19:45:40 -0400 Subject: [PATCH 2/2] dbt-utils dependency --- packages.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 packages.yml diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..30f8e9f --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: fishtown-analytics/dbt_utils + version: ">=0.1.10"