Skip to content

Commit

Permalink
Enable setting current value of dbt_valid_to (#321)
Browse files Browse the repository at this point in the history
  • Loading branch information
gshank authored Oct 10, 2024
1 parent 10e6d59 commit 4df6e54
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 4 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240927-134248.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Enable setting current value of dbt_valid_to
time: 2024-09-27T13:42:48.654556-04:00
custom:
Author: gshank
Issue: "320"
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,13 @@
{{ strategy.unique_key }} as dbt_unique_key

from {{ target_relation }}
where {{ columns.dbt_valid_to }} is null
where
{% if config.get('dbt_valid_to_current') %}
{# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #}
( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null)
{% else %}
{{ columns.dbt_valid_to }} is null
{% endif %}

),

Expand All @@ -64,7 +70,7 @@
{{ strategy.unique_key }} as dbt_unique_key,
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},
{{ get_dbt_valid_to_current(strategy, columns) }},
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }}

from snapshot_query
Expand Down Expand Up @@ -166,7 +172,7 @@
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }},
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}
{{ get_dbt_valid_to_current(strategy, columns) }}
from (
{{ sql }}
) sbq
Expand Down Expand Up @@ -210,3 +216,9 @@
{% endif %}
{% endif %}
{% endmacro %}

{% macro get_dbt_valid_to_current(strategy, columns) %}
{% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %}
coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})
as {{ columns.dbt_valid_to }}
{% endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@
on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}

when matched
and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
{% if config.get("dbt_valid_to_current") %}
and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or
DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null)
{% else %}
and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
{% endif %}
and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
then update
set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
Expand Down

0 comments on commit 4df6e54

Please sign in to comment.