Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable SourceScanOptimizer in query rendering tests #1306

Merged
merged 2 commits into from
Jun 26, 2024
Merged

Enable SourceScanOptimizer in query rendering tests #1306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fc175e
Enable SourceScanOptimizer in query rendering tests
tlento Jun 25, 2024
61de816
Update SQL engine snapshots
tlento Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion tests_metricflow/query_rendering/compare_rendered_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ def render_and_check(
)

# Run dataflow -> sql conversion with all optimizers
optimizations = (DataflowPlanOptimization.PREDICATE_PUSHDOWN,)
optimizations = (
DataflowPlanOptimization.SOURCE_SCAN,
DataflowPlanOptimization.PREDICATE_PUSHDOWN,
)
if is_distinct_values_plan:
optimized_plan = dataflow_plan_builder.build_plan_for_distinct_values(query_spec, optimizations=optimizations)
else:
Expand Down
55 changes: 14 additions & 41 deletions ...erived_metric_rendering.py/SqlQueryPlan/BigQuery/test_derived_metric__plan0_optimized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,22 @@ SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred_bookings_pct
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day) AS metric_time__day
, MAX(subq_15.ref_bookings) AS ref_bookings
, MAX(subq_20.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_15
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_18
GROUP BY
metric_time__day
) subq_20
ON
subq_15.metric_time__day = subq_20.metric_time__day
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_21
) subq_15
95 changes: 25 additions & 70 deletions ...metric_rendering.py/SqlQueryPlan/BigQuery/test_nested_derived_metric__plan0_optimized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,103 +5,58 @@ SELECT
FROM (
-- Combine Aggregated Outputs
SELECT
COALESCE(subq_34.metric_time__day, subq_39.metric_time__day, subq_44.metric_time__day) AS metric_time__day
, MAX(subq_34.non_referred) AS non_referred
, MAX(subq_39.instant) AS instant
, MAX(subq_44.bookings) AS bookings
COALESCE(subq_28.metric_time__day, subq_33.metric_time__day) AS metric_time__day
, MAX(subq_28.non_referred) AS non_referred
, MAX(subq_33.instant) AS instant
, MAX(subq_33.bookings) AS bookings
FROM (
-- Compute Metrics via Expressions
SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_27.metric_time__day, subq_32.metric_time__day) AS metric_time__day
, MAX(subq_27.ref_bookings) AS ref_bookings
, MAX(subq_32.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_25
GROUP BY
metric_time__day
) subq_27
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_30
GROUP BY
metric_time__day
) subq_32
ON
subq_27.metric_time__day = subq_32.metric_time__day
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_25
GROUP BY
metric_time__day
) subq_33
) subq_34
) subq_27
) subq_28
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(instant_bookings) AS instant
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['instant_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_37
GROUP BY
metric_time__day
) subq_39
ON
subq_34.metric_time__day = subq_39.metric_time__day
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
-- Pass Only Elements: ['instant_bookings', 'bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_42
) subq_31
GROUP BY
metric_time__day
) subq_44
) subq_33
ON
COALESCE(subq_34.metric_time__day, subq_39.metric_time__day) = subq_44.metric_time__day
subq_28.metric_time__day = subq_33.metric_time__day
GROUP BY
metric_time__day
) subq_45
) subq_34
52 changes: 13 additions & 39 deletions ...erived_metric_rendering.py/SqlQueryPlan/BigQuery/test_nested_filters__plan0_optimized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,37 @@ FROM (
-- Combine Aggregated Outputs
SELECT
MAX(subq_45.average_booking_value) AS average_booking_value
, MAX(subq_57.bookings) AS bookings
, MAX(subq_64.booking_value) AS booking_value
, MAX(subq_45.bookings) AS bookings
, MAX(subq_52.booking_value) AS booking_value
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['average_booking_value',]
-- Pass Only Elements: ['average_booking_value', 'bookings']
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
AVG(average_booking_value) AS average_booking_value
, SUM(bookings) AS bookings
FROM (
-- Join Standard Outputs
-- Pass Only Elements: ['average_booking_value', 'listing__is_lux_latest', 'booking__is_instant']
-- Pass Only Elements: ['average_booking_value', 'bookings', 'listing__is_lux_latest', 'booking__is_instant']
SELECT
listings_latest_src_28000.is_lux AS listing__is_lux_latest
, subq_36.bookings AS bookings
, subq_36.average_booking_value AS average_booking_value
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['average_booking_value', 'booking__is_instant', 'listing']
-- Pass Only Elements: ['average_booking_value', 'bookings', 'booking__is_instant', 'listing']
SELECT
listing
, bookings
, average_booking_value
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
SELECT
listing_id AS listing
, is_instant AS booking__is_instant
, 1 AS bookings
, booking_value AS average_booking_value
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_34
Expand All @@ -48,36 +52,6 @@ FROM (
) subq_41
WHERE listing__is_lux_latest
) subq_45
CROSS JOIN (
-- Join Standard Outputs
-- Pass Only Elements: ['bookings', 'listing__is_lux_latest', 'booking__is_instant']
-- Pass Only Elements: ['bookings',]
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
SUM(subq_49.bookings) AS bookings
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['bookings', 'booking__is_instant', 'listing']
SELECT
listing
, bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
SELECT
listing_id AS listing
, is_instant AS booking__is_instant
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_47
WHERE booking__is_instant
) subq_49
LEFT OUTER JOIN
***************************.dim_listings_latest listings_latest_src_28000
ON
subq_49.listing = listings_latest_src_28000.listing_id
) subq_57
CROSS JOIN (
-- Constrain Output with WHERE
-- Pass Only Elements: ['booking_value', 'booking__is_instant']
Expand All @@ -93,8 +67,8 @@ FROM (
is_instant AS booking__is_instant
, booking_value
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_59
) subq_47
WHERE booking__is_instant
) subq_64
) subq_65
) subq_66
) subq_52
) subq_53
) subq_54
57 changes: 15 additions & 42 deletions ...ived_metric_rendering.py/SqlQueryPlan/Databricks/test_derived_metric__plan0_optimized.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,22 @@ SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred_bookings_pct
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day) AS metric_time__day
, MAX(subq_15.ref_bookings) AS ref_bookings
, MAX(subq_20.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_15
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_18
GROUP BY
metric_time__day
) subq_20
ON
subq_15.metric_time__day = subq_20.metric_time__day
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day)
) subq_21
metric_time__day
) subq_15
Loading
Loading