Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP - SCD bug fix #1515

Closed
wants to merge 2 commits into from
Closed

WIP - SCD bug fix #1515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
088e4d3
WIP - SCD bug fix
courtneyholcomb Nov 8, 2024
a8d69c2
WIP
courtneyholcomb Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions metricflow-semantics/metricflow_semantics/specs/x.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
SELECT
COUNT(DISTINCT bnr_user_count) AS bnr_user_count
, COUNT(DISTINCT bnr_external_transaction_count) AS bnr_transaction_count
, SUM(complimentary_item_cost) AS complimentary_item_cost
FROM (
SELECT
DATETIME_TRUNC(sem_bnr_transactions_src_10000.transaction_time_est, day) AS metric_time__day
, sem_user_unit_historical_src_10000.property_id AS user_id__property_id
, sem_bnr_transactions_src_10000.category AS bnr_transaction_id__category
, sem_bnr_transactions_src_10000.transaction_id AS bnr_external_transaction_count
, sem_bnr_transactions_src_10000.user_id AS bnr_user_count
, sem_bnr_transactions_src_10000.complimentary_item_cost AS complimentary_item_cost
FROM `bilt-prod`.`analytics`.`stg_all_bnr_transactions` sem_bnr_transactions_src_10000
LEFT OUTER JOIN
`bilt-prod`.`analytics`.`stg_user_units_historical` sem_user_unit_historical_src_10000
ON
(
sem_bnr_transactions_src_10000.user_id = sem_user_unit_historical_src_10000.user_id
) AND (
(
DATETIME_TRUNC(sem_bnr_transactions_src_10000.transaction_time_est, day) >= sem_user_unit_historical_src_10000.user_unit_create_time_est
) AND (
(
DATETIME_TRUNC(sem_bnr_transactions_src_10000.transaction_time_est, day) < sem_user_unit_historical_src_10000.user_unit_delete_time_est
) OR (
sem_user_unit_historical_src_10000.user_unit_delete_time_est IS NULL
)
)
)
) subq_6
WHERE (bnr_transaction_id__category = 'Fitness' ) AND (user_id__property_id = 378182 and metric_time__day>= '2024-01-01')
LIMIT 100
2 changes: 1 addition & 1 deletion .../metricflow_semantics/test_helpers/semantic_manifest_yamls/scd_manifest/scd_bookings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ semantic_model:
- name: ds
type: time
type_params:
time_granularity: day
time_granularity: second
- name: ds_partitioned
type: time
is_partition: true
Expand Down
4 changes: 2 additions & 2 deletions .../metricflow_semantics/test_helpers/semantic_manifest_yamls/scd_manifest/scd_listings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ semantic_model:
type: time
expr: active_from
type_params:
time_granularity: day
time_granularity: second
validity_params:
is_start: True
- name: window_end
type: time
expr: active_to
type_params:
time_granularity: day
time_granularity: second
validity_params:
is_end: True
- name: country
Expand Down
1 change: 1 addition & 0 deletions metricflow/dataset/convert_semantic_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ def _convert_time_dimension(
)
time_dimension_instances.append(time_dimension_instance)

# TODO: figure out how to repro this bug https://dbt-labs.slack.com/archives/C06UEFM66P5/p1730317825261629
# Until we support minimal granularities, we cannot truncate for
# any time dimension used as part of a validity window, since a validity window might
# be stored in seconds while we would truncate to daily.
Expand Down
18 changes: 9 additions & 9 deletions tests_metricflow/fixtures/source_table_snapshots/simple_model/dim_listings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ table_snapshot:
- name: user_id
type: STRING
rows:
- ["2020-01-01", "2020-01-02", "l3141592", "us", "2", "True", "u0004114"]
- ["2020-01-01", "2020-01-02", "l2718281", "cote d'ivoire", "4", "True", "u0005432"]
- ["2020-01-01", "2020-01-02", "l9658588-incomplete", null, null, null, "u1004114"]
- ["2020-01-02", null, "l3141592", "us", "3", "True", "u0004114"]
- ["2020-01-01 00:00:11", "2020-01-02 00:00:11", "l3141592", "us", "2", "True", "u0004114"]
- ["2020-01-01 00:00:11", "2020-01-02 00:00:11", "l2718281", "cote d'ivoire", "4", "True", "u0005432"]
- ["2020-01-01 00:00:11", "2020-01-02 00:00:11", "l9658588-incomplete", null, null, null, "u1004114"]
- ["2020-01-02 00:00:11", null, "l3141592", "us", "3", "True", "u0004114"]
# This cote_divoire property changed hands to a person from Maryland who considers it not lux
- ["2020-01-02", null, "l2718281", "cote d'ivoire", "4", "False", "u0003154"]
- ["2020-01-02", null, "l5948301", "us", "5", "True", "u0004114"]
- ["2020-01-02", null, "l9658588-incomplete", "us", null, null, "u1004114"]
- ["2020-01-02", null, "l8912456-incomplete", null, null, null, "u1004114"]
- ["2020-01-02", null, "l7891283-incomplete", "ca", null, "False", "u1004114"]
- ["2020-01-02 00:00:11", null, "l2718281", "cote d'ivoire", "4", "False", "u0003154"]
- ["2020-01-02 00:00:11", null, "l5948301", "us", "5", "True", "u0004114"]
- ["2020-01-02 00:00:11", null, "l9658588-incomplete", "us", null, null, "u1004114"]
- ["2020-01-02 00:00:11", null, "l8912456-incomplete", null, null, null, "u1004114"]
- ["2020-01-02 00:00:11", null, "l7891283-incomplete", "ca", null, "False", "u1004114"]
92 changes: 46 additions & 46 deletions tests_metricflow/fixtures/source_table_snapshots/simple_model/fct_bookings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,49 +20,49 @@ table_snapshot:
- name: referrer_id
type: STRING
rows:
- ["2019-12-01", "2019-12-01", "2019-12-01", "u0003452", "u0004114", "l5948301", "951.23", "False", "u0003141"]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "797.42", "False", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "808.14", "True", "u0003141"]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0003141", "u0003154", "l2718281", "531.99", "True", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0003154", "u0003141", "l2718281", "285.89", "False", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "714.85", "True", "u0005432"]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "352.42", "False", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "444.14", "True", "u0003141"]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0003141", "u0003154", "l2718281", "415.99", "True", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0003154", "u0003141", "l2718281", "578.89", "False", null]
- ["2019-12-18", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "768.85", "True", "u0005432"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "368.42", "True", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "521.14", "True", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l3141592", "581.99", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0005432", "u0003452", "l2718281", "753.85", "True", "u0003452"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0003452", "u0005432", "l3141592", "754.89", "False", "u0003141"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0003452", "u0004114", "l5948301", "152.23", "False", "u0003141"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "532.42", "True", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "258.14", "True", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l3141592", "753.99", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0005432", "u0003452", "l2718281", "852.85", "True", "u0003452"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0003452", "u0005432", "l3141592", "728.89", "False", "u0003141"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u0003452", "u0004114", "l5948301", "951.23", "False", "u0003141"]
- ["2019-12-19", "2019-12-19", "2019-12-20", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2019-12-20", "2019-12-20", "2019-12-21", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
- ["2019-12-20", "2019-12-20", "2019-12-21", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
- ["2020-01-01", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "742.42", "False", null]
- ["2020-01-01", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "441.14", "True", "u0003141"]
- ["2020-01-01", "2020-01-01", "2020-01-02", "u0003141", "u0003154", "l2718281", "499.99", "True", null]
- ["2020-01-01", "2020-01-01", "2020-01-02", "u0003154", "u0003141", "l2718281", "719.89", "False", null]
- ["2020-01-01", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "319.85", "True", "u0005432"]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l2718281", "442.42", "True", null]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l2718281", "241.14", "True", null]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l3141592", "799.99", "False", null]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0005432", "u0003452", "l2718281", "319.85", "True", "u0003452"]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0003452", "u0005432", "l3141592", "519.89", "False", "u0003141"]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u0003452", "u0004114", "l5948301", "332.23", "False", "u0003141"]
- ["2020-01-02", "2020-01-02", "2020-01-03", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2020-01-03", "2020-01-03", "2020-01-04", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
- ["2019-12-01 00:00:00", "2019-12-01", "2019-12-01", "u0003452", "u0004114", "l5948301", "951.23", "False", "u0003141"]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "797.42", "False", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "808.14", "True", "u0003141"]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0003141", "u0003154", "l2718281", "531.99", "True", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0003154", "u0003141", "l2718281", "285.89", "False", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "714.85", "True", "u0005432"]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "352.42", "False", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "444.14", "True", "u0003141"]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0003141", "u0003154", "l2718281", "415.99", "True", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0003154", "u0003141", "l2718281", "578.89", "False", null]
- ["2019-12-18 00:00:00", "2019-12-18", "2019-12-19", "u0004114", "u0003141", "l3141592", "768.85", "True", "u0005432"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "368.42", "True", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "521.14", "True", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l3141592", "581.99", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0005432", "u0003452", "l2718281", "753.85", "True", "u0003452"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0003452", "u0005432", "l3141592", "754.89", "False", "u0003141"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0003452", "u0004114", "l5948301", "152.23", "False", "u0003141"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "532.42", "True", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l2718281", "258.14", "True", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u0003141", "l3141592", "753.99", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0005432", "u0003452", "l2718281", "852.85", "True", "u0003452"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0003452", "u0005432", "l3141592", "728.89", "False", "u0003141"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u0003452", "u0004114", "l5948301", "951.23", "False", "u0003141"]
- ["2019-12-19 00:00:00", "2019-12-19", "2019-12-20", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2019-12-20 00:00:00", "2019-12-20", "2019-12-21", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
- ["2019-12-20 00:00:00", "2019-12-20", "2019-12-21", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
- ["2020-01-01 00:00:00", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "742.42", "False", null]
- ["2020-01-01 00:00:00", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "441.14", "True", "u0003141"]
- ["2020-01-01 00:00:00", "2020-01-01", "2020-01-02", "u0003141", "u0003154", "l2718281", "499.99", "True", null]
- ["2020-01-01 00:00:00", "2020-01-01", "2020-01-02", "u0003154", "u0003141", "l2718281", "719.89", "False", null]
- ["2020-01-01 00:00:00", "2020-01-01", "2020-01-02", "u0004114", "u0003141", "l3141592", "319.85", "True", "u0005432"]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l2718281", "442.42", "True", null]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l2718281", "241.14", "True", null]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0004114", "u0003141", "l3141592", "799.99", "False", null]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0005432", "u0003452", "l2718281", "319.85", "True", "u0003452"]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0004114", "u1004114", "l9658588-incomplete", "0.0", "False", null]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0004114", "u1004114", "l8912456-incomplete", "0.0", "False", null]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0003452", "u0005432", "l3141592", "519.89", "False", "u0003141"]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u0003452", "u0004114", "l5948301", "332.23", "False", "u0003141"]
- ["2020-01-02 00:00:00", "2020-01-02", "2020-01-03", "u1003452", "u1004114", "no_such_listing", "0.0", "False", null]
- ["2020-01-03 00:00:00", "2020-01-03", "2020-01-04", "u0003452", "u0004114", "l5948301", "0.0", "False", null]
7 changes: 5 additions & 2 deletions tests_metricflow/query_rendering/test_query_rendering.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,11 +298,14 @@ def test_join_to_scd_dimension(
"""Tests conversion of a plan using a dimension with a validity window inside a measure constraint."""
query_spec = scd_query_parser.parse_and_validate_query(
metric_names=("family_bookings",),
group_by_names=(METRIC_TIME_ELEMENT_NAME,),
# group_by_names=(METRIC_TIME_ELEMENT_NAME,),
where_constraints=[
PydanticWhereFilter(
where_sql_template="{{ Dimension('listing__capacity') }} > 2",
)
),
PydanticWhereFilter(
where_sql_template="{{ TimeDimension('metric_time') }} > '2020-01-01'",
),
],
).query_spec

Expand Down
Loading
Loading