Skip to content

Commit

Permalink
Enable SourceScanOptimizer in query rendering tests (#1306)
Browse files Browse the repository at this point in the history 
Enable SourceScanOptimizer in query rendering tests

The query rendering test suite recently added support for
DataflowPlanOptimizer usage, but that only included the
PredicatePushdownOptimizer to support development.

Now that we are getting ready to release we need to see how the
queries come out with all optimizers in place, so we add the
missing SourceScanOptimizer to the set.

Update SQL engine snapshots
  • Loading branch information
@tlento
tlento authored Jun 26, 2024
1 parent 9f1af9e commit 3fb97c8
Show file tree
Hide file tree
Showing 43 changed files with 677 additions and 1,836 deletions.
5 changes: 4 additions & 1 deletion tests_metricflow/query_rendering/compare_rendered_query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ def render_and_check(
)

# Run dataflow -> sql conversion with all optimizers
optimizations = (DataflowPlanOptimization.PREDICATE_PUSHDOWN,)
optimizations = (
DataflowPlanOptimization.SOURCE_SCAN,
DataflowPlanOptimization.PREDICATE_PUSHDOWN,
)
if is_distinct_values_plan:
optimized_plan = dataflow_plan_builder.build_plan_for_distinct_values(query_spec, optimizations=optimizations)
else:
Expand Down
55 changes: 14 additions & 41 deletions ...erived_metric_rendering.py/SqlQueryPlan/BigQuery/test_derived_metric__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,22 @@ SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred_bookings_pct
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day) AS metric_time__day
, MAX(subq_15.ref_bookings) AS ref_bookings
, MAX(subq_20.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_15
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_18
GROUP BY
metric_time__day
) subq_20
ON
subq_15.metric_time__day = subq_20.metric_time__day
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_21
) subq_15
95 changes: 25 additions & 70 deletions ...metric_rendering.py/SqlQueryPlan/BigQuery/test_nested_derived_metric__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,103 +5,58 @@ SELECT
FROM (
-- Combine Aggregated Outputs
SELECT
COALESCE(subq_34.metric_time__day, subq_39.metric_time__day, subq_44.metric_time__day) AS metric_time__day
, MAX(subq_34.non_referred) AS non_referred
, MAX(subq_39.instant) AS instant
, MAX(subq_44.bookings) AS bookings
COALESCE(subq_28.metric_time__day, subq_33.metric_time__day) AS metric_time__day
, MAX(subq_28.non_referred) AS non_referred
, MAX(subq_33.instant) AS instant
, MAX(subq_33.bookings) AS bookings
FROM (
-- Compute Metrics via Expressions
SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_27.metric_time__day, subq_32.metric_time__day) AS metric_time__day
, MAX(subq_27.ref_bookings) AS ref_bookings
, MAX(subq_32.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_25
GROUP BY
metric_time__day
) subq_27
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_30
GROUP BY
metric_time__day
) subq_32
ON
subq_27.metric_time__day = subq_32.metric_time__day
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_25
GROUP BY
metric_time__day
) subq_33
) subq_34
) subq_27
) subq_28
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(instant_bookings) AS instant
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['instant_bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_37
GROUP BY
metric_time__day
) subq_39
ON
subq_34.metric_time__day = subq_39.metric_time__day
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
-- Pass Only Elements: ['instant_bookings', 'bookings', 'metric_time__day']
SELECT
DATETIME_TRUNC(ds, day) AS metric_time__day
, 1 AS bookings
, CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_42
) subq_31
GROUP BY
metric_time__day
) subq_44
) subq_33
ON
COALESCE(subq_34.metric_time__day, subq_39.metric_time__day) = subq_44.metric_time__day
subq_28.metric_time__day = subq_33.metric_time__day
GROUP BY
metric_time__day
) subq_45
) subq_34
52 changes: 13 additions & 39 deletions ...erived_metric_rendering.py/SqlQueryPlan/BigQuery/test_nested_filters__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,37 @@ FROM (
-- Combine Aggregated Outputs
SELECT
MAX(subq_45.average_booking_value) AS average_booking_value
, MAX(subq_57.bookings) AS bookings
, MAX(subq_64.booking_value) AS booking_value
, MAX(subq_45.bookings) AS bookings
, MAX(subq_52.booking_value) AS booking_value
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['average_booking_value',]
-- Pass Only Elements: ['average_booking_value', 'bookings']
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
AVG(average_booking_value) AS average_booking_value
, SUM(bookings) AS bookings
FROM (
-- Join Standard Outputs
-- Pass Only Elements: ['average_booking_value', 'listing__is_lux_latest', 'booking__is_instant']
-- Pass Only Elements: ['average_booking_value', 'bookings', 'listing__is_lux_latest', 'booking__is_instant']
SELECT
listings_latest_src_28000.is_lux AS listing__is_lux_latest
, subq_36.bookings AS bookings
, subq_36.average_booking_value AS average_booking_value
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['average_booking_value', 'booking__is_instant', 'listing']
-- Pass Only Elements: ['average_booking_value', 'bookings', 'booking__is_instant', 'listing']
SELECT
listing
, bookings
, average_booking_value
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
SELECT
listing_id AS listing
, is_instant AS booking__is_instant
, 1 AS bookings
, booking_value AS average_booking_value
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_34
Expand All @@ -48,36 +52,6 @@ FROM (
) subq_41
WHERE listing__is_lux_latest
) subq_45
CROSS JOIN (
-- Join Standard Outputs
-- Pass Only Elements: ['bookings', 'listing__is_lux_latest', 'booking__is_instant']
-- Pass Only Elements: ['bookings',]
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
SUM(subq_49.bookings) AS bookings
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['bookings', 'booking__is_instant', 'listing']
SELECT
listing
, bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
SELECT
listing_id AS listing
, is_instant AS booking__is_instant
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_47
WHERE booking__is_instant
) subq_49
LEFT OUTER JOIN
***************************.dim_listings_latest listings_latest_src_28000
ON
subq_49.listing = listings_latest_src_28000.listing_id
) subq_57
CROSS JOIN (
-- Constrain Output with WHERE
-- Pass Only Elements: ['booking_value', 'booking__is_instant']
Expand All @@ -93,8 +67,8 @@ FROM (
is_instant AS booking__is_instant
, booking_value
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_59
) subq_47
WHERE booking__is_instant
) subq_64
) subq_65
) subq_66
) subq_52
) subq_53
) subq_54
57 changes: 15 additions & 42 deletions ...ived_metric_rendering.py/SqlQueryPlan/Databricks/test_derived_metric__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,22 @@ SELECT
metric_time__day
, (bookings - ref_bookings) * 1.0 / bookings AS non_referred_bookings_pct
FROM (
-- Combine Aggregated Outputs
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day) AS metric_time__day
, MAX(subq_15.ref_bookings) AS ref_bookings
, MAX(subq_20.bookings) AS bookings
metric_time__day
, SUM(referred_bookings) AS ref_bookings
, SUM(bookings) AS bookings
FROM (
-- Aggregate Measures
-- Compute Metrics via Expressions
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'bookings', 'metric_time__day']
SELECT
metric_time__day
, SUM(referred_bookings) AS ref_bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['referred_bookings', 'metric_time__day']
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
metric_time__day
) subq_15
FULL OUTER JOIN (
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(bookings) AS bookings
FROM (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['bookings', 'metric_time__day']
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_18
GROUP BY
metric_time__day
) subq_20
ON
subq_15.metric_time__day = subq_20.metric_time__day
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
, CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_13
GROUP BY
COALESCE(subq_15.metric_time__day, subq_20.metric_time__day)
) subq_21
metric_time__day
) subq_15
Loading

0 comments on commit 3fb97c8

Please sign in to comment.