Disable predicate pushdown optimizer (#1400)

Disable predicate pushdown optimizer. This has some bugs that we haven't investigated yet. It has been disabled in our servers, so it should be disabled in MetricFlow by default, too, to ensure where we're doing all our testing matches what's happening in production.
dbt-labs · Sep 10, 2024 · 024ca33 · 024ca33
1 parent e6c2196
commit 024ca33
Show file tree

Hide file tree

Showing 122 changed files with 1,257 additions and 1,345 deletions.
diff --git a/metricflow/dataflow/optimizer/dataflow_optimizer_factory.py b/metricflow/dataflow/optimizer/dataflow_optimizer_factory.py
@@ -29,6 +29,14 @@ def all_optimizations() -> FrozenSet[DataflowPlanOptimization]:
         """Convenience method for getting a set of all available optimizations."""
         return frozenset((DataflowPlanOptimization.SOURCE_SCAN, DataflowPlanOptimization.PREDICATE_PUSHDOWN))
 
+    @staticmethod
+    def enabled_optimizations() -> FrozenSet[DataflowPlanOptimization]:
+        """Set of DataflowPlanOptimization that are currently enabled.
+
+        Predicate pushdown optimizer is currently disabled.
+        """
+        return frozenset((DataflowPlanOptimization.SOURCE_SCAN,))
+
 
 class DataflowPlanOptimizerFactory:
     """Factory class for initializing an enumerated set of optimizers.

diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py
@@ -111,7 +111,7 @@ class MetricFlowQueryRequest:
     order_by: Optional[Sequence[OrderByQueryParameter]] = None
     min_max_only: bool = False
     sql_optimization_level: SqlQueryOptimizationLevel = SqlQueryOptimizationLevel.O4
-    dataflow_plan_optimizations: FrozenSet[DataflowPlanOptimization] = DataflowPlanOptimization.all_optimizations()
+    dataflow_plan_optimizations: FrozenSet[DataflowPlanOptimization] = DataflowPlanOptimization.enabled_optimizations()
     query_type: MetricFlowQueryType = MetricFlowQueryType.METRIC
 
     @staticmethod
@@ -128,7 +128,9 @@ def create_with_random_request_id(  # noqa: D102
         order_by_names: Optional[Sequence[str]] = None,
         order_by: Optional[Sequence[OrderByQueryParameter]] = None,
         sql_optimization_level: SqlQueryOptimizationLevel = SqlQueryOptimizationLevel.O4,
-        dataflow_plan_optimizations: FrozenSet[DataflowPlanOptimization] = DataflowPlanOptimization.all_optimizations(),
+        dataflow_plan_optimizations: FrozenSet[
+            DataflowPlanOptimization
+        ] = DataflowPlanOptimization.enabled_optimizations(),
         query_type: MetricFlowQueryType = MetricFlowQueryType.METRIC,
         min_max_only: bool = False,
     ) -> MetricFlowQueryRequest:

diff --git a/tests_metricflow/query_rendering/compare_rendered_query.py b/tests_metricflow/query_rendering/compare_rendered_query.py
@@ -54,11 +54,11 @@ def render_and_check(
     # Run dataflow -> sql conversion with all optimizers
     if is_distinct_values_plan:
         optimized_plan = dataflow_plan_builder.build_plan_for_distinct_values(
-            query_spec, optimizations=DataflowPlanOptimization.all_optimizations()
+            query_spec, optimizations=DataflowPlanOptimization.enabled_optimizations()
         )
     else:
         optimized_plan = dataflow_plan_builder.build_plan(
-            query_spec, optimizations=DataflowPlanOptimization.all_optimizations()
+            query_spec, optimizations=DataflowPlanOptimization.enabled_optimizations()
         )
     conversion_result = dataflow_to_sql_converter.convert_to_sql_query_plan(
         sql_engine_type=sql_client.sql_engine_type,

diff --git a/...SqlQueryPlan/BigQuery/test_conversion_metric_with_categorical_filter__plan0_optimized.sql b/...SqlQueryPlan/BigQuery/test_conversion_metric_with_categorical_filter__plan0_optimized.sql
@@ -12,7 +12,6 @@ FROM (
     , MAX(subq_30.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -21,12 +20,13 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATETIME_TRUNC(ds, day) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_17
+    ) subq_18
     WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day

diff --git a/...py/SqlQueryPlan/BigQuery/test_conversion_metric_with_time_constraint__plan0_optimized.sql b/...py/SqlQueryPlan/BigQuery/test_conversion_metric_with_time_constraint__plan0_optimized.sql
@@ -10,26 +10,22 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id']
     -- Aggregate Measures
     SELECT
       visit__referrer_id
       , SUM(visits) AS visits
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id']
       SELECT
-        DATETIME_TRUNC(ds, day) AS metric_time__day
-        , referrer_id AS visit__referrer_id
+        referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATETIME_TRUNC(ds, day) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       visit__referrer_id
   ) subq_23

diff --git a/...Plan/BigQuery/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql b/...Plan/BigQuery/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql
@@ -12,8 +12,6 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -22,17 +20,16 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATETIME_TRUNC(ds, day) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATETIME_TRUNC(ds, day) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day
       , visit__referrer_id

diff --git a/...lQueryPlan/Databricks/test_conversion_metric_with_categorical_filter__plan0_optimized.sql b/...lQueryPlan/Databricks/test_conversion_metric_with_categorical_filter__plan0_optimized.sql
@@ -12,7 +12,6 @@ FROM (
     , MAX(subq_30.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -21,12 +20,13 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATE_TRUNC('day', ds) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_17
+    ) subq_18
     WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day

diff --git a/.../SqlQueryPlan/Databricks/test_conversion_metric_with_time_constraint__plan0_optimized.sql b/.../SqlQueryPlan/Databricks/test_conversion_metric_with_time_constraint__plan0_optimized.sql
@@ -10,26 +10,22 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id']
     -- Aggregate Measures
     SELECT
       visit__referrer_id
       , SUM(visits) AS visits
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id']
       SELECT
-        DATE_TRUNC('day', ds) AS metric_time__day
-        , referrer_id AS visit__referrer_id
+        referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATE_TRUNC('day', ds) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       visit__referrer_id
   ) subq_23

diff --git a/...an/Databricks/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql b/...an/Databricks/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql
@@ -12,8 +12,6 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -22,17 +20,16 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATE_TRUNC('day', ds) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATE_TRUNC('day', ds) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day
       , visit__referrer_id

diff --git a/...y/SqlQueryPlan/DuckDB/test_conversion_metric_with_categorical_filter__plan0_optimized.sql b/...y/SqlQueryPlan/DuckDB/test_conversion_metric_with_categorical_filter__plan0_optimized.sql
@@ -12,7 +12,6 @@ FROM (
     , MAX(subq_30.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -21,12 +20,13 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATE_TRUNC('day', ds) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_17
+    ) subq_18
     WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day

diff --git a/...g.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_time_constraint__plan0_optimized.sql b/...g.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_time_constraint__plan0_optimized.sql
@@ -10,26 +10,22 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id']
     -- Aggregate Measures
     SELECT
       visit__referrer_id
       , SUM(visits) AS visits
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id']
       SELECT
-        DATE_TRUNC('day', ds) AS metric_time__day
-        , referrer_id AS visit__referrer_id
+        referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATE_TRUNC('day', ds) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       visit__referrer_id
   ) subq_23

diff --git a/...ryPlan/DuckDB/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql b/...ryPlan/DuckDB/test_conversion_metric_with_window_and_time_constraint__plan0_optimized.sql
@@ -12,8 +12,6 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -22,17 +20,16 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATE_TRUNC('day', ds) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATE_TRUNC('day', ds) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day
       , visit__referrer_id

diff --git a/...SqlQueryPlan/Postgres/test_conversion_metric_with_categorical_filter__plan0_optimized.sql b/...SqlQueryPlan/Postgres/test_conversion_metric_with_categorical_filter__plan0_optimized.sql
@@ -12,7 +12,6 @@ FROM (
     , MAX(subq_30.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
     -- Aggregate Measures
     SELECT
       metric_time__day
@@ -21,12 +20,13 @@ FROM (
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Pass Only Elements: ['visits', 'visit__referrer_id', 'metric_time__day']
       SELECT
         DATE_TRUNC('day', ds) AS metric_time__day
         , referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_17
+    ) subq_18
     WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       metric_time__day

diff --git a/...py/SqlQueryPlan/Postgres/test_conversion_metric_with_time_constraint__plan0_optimized.sql b/...py/SqlQueryPlan/Postgres/test_conversion_metric_with_time_constraint__plan0_optimized.sql
@@ -10,26 +10,22 @@ FROM (
     , MAX(subq_34.buys) AS buys
   FROM (
     -- Constrain Output with WHERE
-    -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
-    -- Pass Only Elements: ['visits', 'visit__referrer_id']
     -- Aggregate Measures
     SELECT
       visit__referrer_id
       , SUM(visits) AS visits
     FROM (
       -- Read Elements From Semantic Model 'visits_source'
       -- Metric Time Dimension 'ds'
+      -- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-02T00:00:00]
+      -- Pass Only Elements: ['visits', 'visit__referrer_id']
       SELECT
-        DATE_TRUNC('day', ds) AS metric_time__day
-        , referrer_id AS visit__referrer_id
+        referrer_id AS visit__referrer_id
         , 1 AS visits
       FROM ***************************.fct_visits visits_source_src_28000
-    ) subq_19
-    WHERE (
-      metric_time__day BETWEEN '2020-01-01' AND '2020-01-02'
-    ) AND (
-      visit__referrer_id = 'ref_id_01'
-    )
+      WHERE DATE_TRUNC('day', ds) BETWEEN '2020-01-01' AND '2020-01-02'
+    ) subq_21
+    WHERE visit__referrer_id = 'ref_id_01'
     GROUP BY
       visit__referrer_id
   ) subq_23