Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bug fix: custom granularity join column gets pruned #1425

Merged
merged 2 commits into from
Sep 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions metricflow/plan_conversion/dataflow_to_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -1441,23 +1441,32 @@ def visit_join_to_custom_granularity_node(self, node: JoinToCustomGranularityNod
if instance.spec == node.time_dimension_spec.with_base_grain():
parent_time_dimension_instance = instance
break
parent_column: Optional[SqlSelectColumn] = None
assert parent_time_dimension_instance, (
"JoinToCustomGranularityNode's expected time_dimension_spec not found in parent dataset instances. "
"This indicates internal misconfiguration."
f"This indicates internal misconfiguration. Expected: {node.time_dimension_spec.with_base_grain}; "
f"Got: {[instance.spec for instance in parent_data_set.instance_set.time_dimension_instances]}"
)
for select_column in parent_data_set.checked_sql_select_node.select_columns:
if select_column.column_alias == parent_time_dimension_instance.associated_column.column_name:
parent_column = select_column
break
assert parent_column, (
"JoinToCustomGranularityNode's expected time_dimension_spec not found in parent columns. "
f"This indicates internal misconfiguration. Expected: "
f"{parent_time_dimension_instance.associated_column.column_name}; Got: "
f"{[column.column_alias for column in parent_data_set.checked_sql_select_node.select_columns]}"
)

# Build join expression.
time_spine_alias = self._next_unique_table_alias()
custom_granularity_name = node.time_dimension_spec.time_granularity.name
time_spine_source = self._get_time_spine_for_custom_granularity(custom_granularity_name)
left_expr_for_join: SqlExpressionNode = SqlColumnReferenceExpression.from_table_and_column_names(
table_alias=parent_alias, column_name=parent_time_dimension_instance.associated_column.column_name
)
join_description = SqlJoinDescription(
right_source=SqlTableNode.create(sql_table=time_spine_source.spine_table),
right_source_alias=time_spine_alias,
on_condition=SqlComparisonExpression.create(
left_expr=left_expr_for_join,
left_expr=parent_column.expr,
comparison=SqlComparison.EQUALS,
right_expr=SqlColumnReferenceExpression.from_table_and_column_names(
table_alias=time_spine_alias, column_name=time_spine_source.base_column
Expand Down
2 changes: 2 additions & 0 deletions metricflow/sql/optimizer/column_pruner.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ def visit_select_statement_node(self, node: SqlSelectStatementNode) -> SqlQueryP
or select_column in node.group_bys
or node.distinct
)
# TODO: don't prune columns used in join condition! Tricky to derive since the join condition can be any
# SqlExpressionNode.

if len(pruned_select_columns) == 0:
raise RuntimeError("All columns have been pruned - this indicates an bug in the pruner or in the inputs.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATETIME_TRUNC(time_spine_src_28006.ds, day) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATETIME_TRUNC(time_spine_src_28006.ds, day) = subq_4.ds
GROUP BY
metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATETIME_TRUNC(bookings_source_src_28000.ds, day) = subq_0.ds
) subq_1
GROUP BY
booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATETIME_TRUNC(bookings_source_src_28000.ds, day) = subq_2.ds
GROUP BY
booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_0.ds
) subq_1
GROUP BY
subq_1.booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_2.ds
GROUP BY
subq_2.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_0.ds
) subq_1
GROUP BY
subq_1.booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_2.ds
GROUP BY
subq_2.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_0.ds
) subq_1
GROUP BY
subq_1.booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_2.ds
GROUP BY
subq_2.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_0.ds
) subq_1
GROUP BY
subq_1.booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_2.ds
GROUP BY
subq_2.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_0
ON
bookings_source_src_28000.booking__ds__day = subq_0.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_0.ds
) subq_1
GROUP BY
subq_1.booking__ds__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.fct_bookings bookings_source_src_28000
LEFT OUTER JOIN
***************************.mf_time_spine subq_2
ON
bookings_source_src_28000.booking__ds__day = subq_2.ds
DATE_TRUNC('day', bookings_source_src_28000.ds) = subq_2.ds
GROUP BY
subq_2.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_0.metric_time__day = subq_4.ds
subq_3.metric_time__day = subq_4.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_5
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ CROSS JOIN
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
subq_7.metric_time__day = subq_11.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ FROM (
LEFT OUTER JOIN
***************************.mf_time_spine subq_1
ON
subq_0.metric_time__day = subq_1.ds
subq_0.ds__day = subq_1.ds
) subq_2
GROUP BY
subq_2.metric_time__martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
subq_3.metric_time__day = subq_4.ds
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
GROUP BY
subq_4.martian_day
Loading
Loading