From 04fe56835d5b9c55049e97934c380a59c6ce147d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sat, 5 Oct 2024 00:12:42 -0700 Subject: [PATCH] add druid.expressions.allowVectorizeFallback and default to false (#17248) changes: adds ExpressionProcessing.allowVectorizeFallback() and ExpressionProcessingConfig.allowVectorizeFallback(), defaulting to false until few remaining bugs can be fixed (mostly complex types and some odd interactions with mixed types) add cannotVectorizeUnlessFallback functions to make it easy to toggle the default of this config, and easy to know what to delete when we remove it in the future --- docs/configuration/index.md | 1 - .../hll/sql/HllSketchSqlAggregatorTest.java | 2 + .../sql/ThetaSketchSqlAggregatorTest.java | 2 + .../java/org/apache/druid/math/expr/Expr.java | 7 +++ .../druid/math/expr/ExprMacroTable.java | 2 +- .../druid/math/expr/ExpressionProcessing.java | 18 ++++++-- .../math/expr/ExpressionProcessingConfig.java | 16 ++++++- .../druid/math/expr/FunctionalExpr.java | 6 +-- .../druid/math/expr/VectorExprSanityTest.java | 18 +++++--- .../timeseries/TimeseriesQueryRunnerTest.java | 10 +++++ .../druid/segment/filter/BaseFilterTest.java | 14 ++++++ .../segment/filter/EqualityFilterTests.java | 14 +++--- .../virtual/ExpressionPlannerTest.java | 40 ++++++++++++----- .../sql/calcite/BaseCalciteQueryTest.java | 11 ++++- .../sql/calcite/CalciteArraysQueryTest.java | 7 +++ .../sql/calcite/CalciteJoinQueryTest.java | 1 + .../CalciteLookupFunctionQueryTest.java | 3 ++ .../CalciteMultiValueStringQueryTest.java | 7 +++ .../calcite/CalciteNestedDataQueryTest.java | 2 + .../druid/sql/calcite/CalciteQueryTest.java | 45 +++++++++++++++++++ .../sql/calcite/CalciteSelectQueryTest.java | 6 +++ .../sql/calcite/CalciteSubqueryTest.java | 2 + .../calcite/CalciteTimeBoundaryQueryTest.java | 1 + .../druid/sql/calcite/DecoupledExtension.java | 27 +++++++---- 24 files changed, 221 insertions(+), 41 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 427c8b8d16b9..ba41bc5ac49a 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -2218,7 +2218,6 @@ Supported query contexts: |Key|Description|Default| |---|-----------|-------| |`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values are either `1` or `0`. This configuration has been deprecated and will be removed in a future release, taking on the `true` behavior. See [expression documentation](../querying/math-expr.md#logical-operator-modes) for more information.|true| -|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays. This configuration has been deprecated and will be removed in a future release, taking on the `true` behavior.|true| ### Router diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index 65451a3323e1..02359a50b62e 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -322,6 +322,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( @Test public void testApproxCountDistinctHllSketch() { + cannotVectorizeUnlessFallback(); final String sql = "SELECT\n" + " SUM(cnt),\n" + " APPROX_COUNT_DISTINCT_DS_HLL(dim2),\n" // uppercase @@ -1138,6 +1139,7 @@ public void testHllEstimateAsVirtualColumnOnNonHllCol() @Test public void testHllEstimateAsVirtualColumnWithGroupByOrderBy() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT" + " HLL_SKETCH_ESTIMATE(hllsketch_dim1), count(*)" diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index 7afd2710ccd0..d82863a8bf15 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -177,6 +177,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( @Test public void testApproxCountDistinctThetaSketch() { + cannotVectorizeUnlessFallback(); final String sql = "SELECT\n" + " SUM(cnt),\n" + " APPROX_COUNT_DISTINCT_DS_THETA(dim2),\n" @@ -1157,6 +1158,7 @@ public void testThetaEstimateAsVirtualColumnOnNonThetaCol() @Test public void testThetaEstimateAsVirtualColumnWithGroupByOrderBy() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT" + " THETA_SKETCH_ESTIMATE(thetasketch_dim1), count(*)" diff --git a/processing/src/main/java/org/apache/druid/math/expr/Expr.java b/processing/src/main/java/org/apache/druid/math/expr/Expr.java index 8fa025cf9145..104cc5b1457d 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/Expr.java @@ -182,6 +182,13 @@ default boolean canVectorize(InputBindingInspector inspector) return false; } + + default boolean canFallbackVectorize(InputBindingInspector inspector, List args) + { + return ExpressionProcessing.allowVectorizeFallback() && + getOutputType(inspector) != null && + inspector.canVectorize(args); + } /** * Possibly convert the {@link Expr} into an optimized, possibly not thread-safe {@link Expr}. Does not convert * child {@link Expr}. Most callers should use {@link Expr#singleThreaded(Expr, InputBindingInspector)} to convert diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java index 699eb8967eb8..40d8ba1112c4 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java @@ -178,7 +178,7 @@ public BindingAnalysis analyzeInputs() @Override public boolean canVectorize(InputBindingInspector inspector) { - return getOutputType(inspector) != null && inspector.canVectorize(args); + return canFallbackVectorize(inspector, args); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java index ae1d5fb297bb..9a4d2ef46942 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java @@ -45,19 +45,25 @@ public class ExpressionProcessing @VisibleForTesting public static void initializeForTests() { - INSTANCE = new ExpressionProcessingConfig(null, null, null); + INSTANCE = new ExpressionProcessingConfig(null, null, null, null); } @VisibleForTesting public static void initializeForStrictBooleansTests(boolean useStrict) { - INSTANCE = new ExpressionProcessingConfig(useStrict, null, null); + INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null); } @VisibleForTesting public static void initializeForHomogenizeNullMultiValueStrings() { - INSTANCE = new ExpressionProcessingConfig(null, null, true); + INSTANCE = new ExpressionProcessingConfig(null, null, true, null); + } + + @VisibleForTesting + public static void initializeForFallback() + { + INSTANCE = new ExpressionProcessingConfig(null, null, null, true); } /** @@ -90,6 +96,12 @@ public static boolean isHomogenizeNullMultiValueStringArrays() return INSTANCE.isHomogenizeNullMultiValueStringArrays(); } + public static boolean allowVectorizeFallback() + { + checkInitialized(); + return INSTANCE.allowVectorizeFallback(); + } + private static void checkInitialized() { // this should only be null in a unit test context, in production this will be injected by the null handling module diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java index daae90ef5341..3d235fe3ddaa 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java @@ -37,6 +37,7 @@ public class ExpressionProcessingConfig // Coerce 'null', '[]', and '[null]' into '[null]' for backwards compat with 0.22 and earlier public static final String HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS = "druid.expressions.homogenizeNullMultiValueStringArrays"; + public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback"; @JsonProperty("useStrictBooleans") private final boolean useStrictBooleans; @@ -47,11 +48,15 @@ public class ExpressionProcessingConfig @JsonProperty("homogenizeNullMultiValueStringArrays") private final boolean homogenizeNullMultiValueStringArrays; + @JsonProperty("allowVectorizeFallback") + private final boolean allowVectorizeFallback; + @JsonCreator public ExpressionProcessingConfig( @JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans, @JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings, - @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays + @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays, + @JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback ) { this.useStrictBooleans = getWithPropertyFallback( @@ -67,6 +72,10 @@ public ExpressionProcessingConfig( homogenizeNullMultiValueStringArrays, HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS ); + this.allowVectorizeFallback = getWithPropertyFallbackFalse( + allowVectorizeFallback, + ALLOW_VECTORIZE_FALLBACK + ); String version = ExpressionProcessingConfig.class.getPackage().getImplementationVersion(); if (version == null || version.contains("SNAPSHOT")) { version = "latest"; @@ -95,6 +104,11 @@ public boolean isHomogenizeNullMultiValueStringArrays() return homogenizeNullMultiValueStringArrays; } + public boolean allowVectorizeFallback() + { + return allowVectorizeFallback; + } + private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property) { return getWithPropertyFallback(value, property, "false"); diff --git a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java index a7319e0f0287..e95140011a78 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java @@ -100,8 +100,7 @@ public ExprEval eval(ObjectBinding bindings) @Override public boolean canVectorize(InputBindingInspector inspector) { - return function.canVectorize(inspector, args) - || (getOutputType(inspector) != null && inspector.canVectorize(args)); + return function.canVectorize(inspector, args) || canFallbackVectorize(inspector, args); } @Override @@ -226,7 +225,8 @@ public ExprEval eval(ObjectBinding bindings) @Override public boolean canVectorize(InputBindingInspector inspector) { - return canVectorizeNative(inspector) || (getOutputType(inspector) != null && inspector.canVectorize(argsExpr)); + return canVectorizeNative(inspector) || + (canFallbackVectorize(inspector, argsExpr) && lambdaExpr.canVectorize(inspector)); } @Override diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index cf513b4ffba8..2da11bb4e29c 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -30,6 +30,7 @@ import org.apache.druid.query.expression.NestedDataExpressions; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; import javax.annotation.Nullable; @@ -264,11 +265,17 @@ public void testStringFns() @Test public void testArrayFns() { - testExpression("array(s1, s2)", types); - testExpression("array(l1, l2)", types); - testExpression("array(d1, d2)", types); - testExpression("array(l1, d2)", types); - testExpression("array(s1, l2)", types); + try { + ExpressionProcessing.initializeForFallback(); + testExpression("array(s1, s2)", types); + testExpression("array(l1, l2)", types); + testExpression("array(d1, d2)", types); + testExpression("array(l1, d2)", types); + testExpression("array(s1, l2)", types); + } + finally { + ExpressionProcessing.initializeForTests(); + } } @Test @@ -284,6 +291,7 @@ public void testCastArraysRoundTrip() @Test public void testJsonFns() { + Assume.assumeTrue(ExpressionProcessing.allowVectorizeFallback()); testExpression("json_object('k1', s1, 'k2', l1)", types); } diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java index 781d9d31023e..9c89d6308525 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -35,6 +35,7 @@ import org.apache.druid.java.util.common.granularity.PeriodGranularity; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.Druids; import org.apache.druid.query.FinalizeResultsQueryRunner; import org.apache.druid.query.MetricsEmittingQueryRunner; @@ -2269,6 +2270,7 @@ public void testTimeSeriesWithFilteredAgg() @Test public void testTimeSeriesWithFilteredAggAndExpressionFilteredAgg() { + cannotVectorizeUnlessFallback(); TimeseriesQuery query = Druids .newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.DATA_SOURCE) @@ -3278,4 +3280,12 @@ protected void cannotVectorize() expectedException.expectMessage("Cannot vectorize!"); } } + + protected void cannotVectorizeUnlessFallback() + { + if (vectorize && !ExpressionProcessing.allowVectorizeFallback()) { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot vectorize!"); + } + } } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index ed4ff921a9c5..965b3517b244 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -48,6 +48,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.Parser; import org.apache.druid.query.QueryContext; @@ -1192,6 +1193,19 @@ protected void assertFilterMatchesSkipVectorize( } } + protected void assertFilterMatchesSkipVectorizeUnlessFallback( + final DimFilter filter, + final List expectedRows + ) + { + final boolean vectorize = ExpressionProcessing.allowVectorizeFallback(); + assertFilterMatches(filter, expectedRows, vectorize); + // test double inverted + if (!StringUtils.toLowerCase(testName).contains("concise")) { + assertFilterMatches(NotDimFilter.of(NotDimFilter.of(filter)), expectedRows, vectorize); + } + } + private void assertFilterMatches( final DimFilter filter, final List expectedRows, diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java index 9e29ff266b1e..c76ea3e6b851 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java @@ -1644,7 +1644,7 @@ public void testArraysAsMvds() "5", .. [null], [123L, 345L], null */ - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( new EqualityFilter( "arrayStringAsMvd", ColumnType.STRING, @@ -1653,7 +1653,7 @@ public void testArraysAsMvds() ), ImmutableList.of("0", "3") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( NotDimFilter.of( new EqualityFilter( "arrayStringAsMvd", @@ -1667,7 +1667,7 @@ public void testArraysAsMvds() : ImmutableList.of("1", "2", "4", "5") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( new EqualityFilter( "arrayLongAsMvd", ColumnType.STRING, @@ -1676,7 +1676,7 @@ public void testArraysAsMvds() ), ImmutableList.of("0", "2") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( NotDimFilter.of( new EqualityFilter( "arrayLongAsMvd", @@ -1690,7 +1690,7 @@ public void testArraysAsMvds() : ImmutableList.of("1", "3", "4", "5") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( new EqualityFilter( "arrayDoubleAsMvd", ColumnType.STRING, @@ -1699,7 +1699,7 @@ public void testArraysAsMvds() ), ImmutableList.of("0", "1") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( NotDimFilter.of( new EqualityFilter( "arrayDoubleAsMvd", @@ -1713,7 +1713,7 @@ public void testArraysAsMvds() : ImmutableList.of("2", "3", "4", "5") ); - assertFilterMatches( + assertFilterMatchesSkipVectorizeUnlessFallback( new EqualityFilter( "arrayConstantAsMvd", ColumnType.STRING, diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java index a0f3d1e8ecc4..3dddbd991c32 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionPlannerTest.java @@ -26,6 +26,7 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.Parser; import org.apache.druid.query.expression.TestExprMacroTable; @@ -1151,8 +1152,7 @@ public void testScalarOutputMultiValueInput() Assert.assertTrue( thePlan.is( ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.NEEDS_APPLIED, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NEEDS_APPLIED ) ); Assert.assertFalse( @@ -1164,6 +1164,7 @@ public void testScalarOutputMultiValueInput() ExpressionPlan.Trait.UNKNOWN_INPUTS ) ); + assertFallbackVectorizable(thePlan); Assert.assertEquals( "array_to_string(map((\"multi_dictionary_string\") -> array_append(\"scalar_string\", \"multi_dictionary_string\"), \"multi_dictionary_string\"), ',')", @@ -1251,8 +1252,7 @@ public void testArrayConstruction() ExpressionPlan thePlan = plan("array(long1, long2)"); Assert.assertTrue( thePlan.is( - ExpressionPlan.Trait.NON_SCALAR_OUTPUT, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NON_SCALAR_OUTPUT ) ); Assert.assertFalse( @@ -1265,6 +1265,7 @@ public void testArrayConstruction() ExpressionPlan.Trait.NON_SCALAR_INPUTS ) ); + assertFallbackVectorizable(thePlan); Assert.assertEquals(ExpressionType.LONG_ARRAY, thePlan.getOutputType()); thePlan = plan("array(long1, double1)"); @@ -1341,10 +1342,11 @@ public void testDictionaryComplexStringOutput() ); Assert.assertTrue( thePlan.is( - ExpressionPlan.Trait.SINGLE_INPUT_SCALAR, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.SINGLE_INPUT_SCALAR ) ); + assertFallbackVectorizable(thePlan); + Assert.assertEquals(ExpressionType.STRING, thePlan.getOutputType()); ColumnCapabilities inferred = thePlan.inferColumnCapabilities( ExpressionType.toColumnType(thePlan.getOutputType()) @@ -1387,8 +1389,7 @@ private static void assertArrayInput(ExpressionPlan thePlan) { Assert.assertTrue( thePlan.is( - ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NON_SCALAR_INPUTS ) ); Assert.assertFalse( @@ -1401,6 +1402,7 @@ private static void assertArrayInput(ExpressionPlan thePlan) ExpressionPlan.Trait.NEEDS_APPLIED ) ); + assertFallbackVectorizable(thePlan); } private static void assertArrayInAndOut(ExpressionPlan thePlan) @@ -1408,8 +1410,7 @@ private static void assertArrayInAndOut(ExpressionPlan thePlan) Assert.assertTrue( thePlan.is( ExpressionPlan.Trait.NON_SCALAR_INPUTS, - ExpressionPlan.Trait.NON_SCALAR_OUTPUT, - ExpressionPlan.Trait.VECTORIZABLE + ExpressionPlan.Trait.NON_SCALAR_OUTPUT ) ); Assert.assertFalse( @@ -1421,6 +1422,25 @@ private static void assertArrayInAndOut(ExpressionPlan thePlan) ExpressionPlan.Trait.NEEDS_APPLIED ) ); + assertFallbackVectorizable(thePlan); + } + + + private static void assertFallbackVectorizable(ExpressionPlan thePlan) + { + if (ExpressionProcessing.allowVectorizeFallback()) { + Assert.assertTrue( + thePlan.is( + ExpressionPlan.Trait.VECTORIZABLE + ) + ); + } else { + Assert.assertFalse( + thePlan.is( + ExpressionPlan.Trait.VECTORIZABLE + ) + ); + } } private static class TestMacroTable extends ExprMacroTable diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 42cf41ff32a2..20c21172f41b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -43,6 +43,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.JoinDataSource; @@ -282,6 +283,7 @@ public static Map getTimeseriesContextWithFloorTime( final boolean useDefault = NullHandling.replaceWithDefault(); public boolean cannotVectorize = false; + public boolean cannotVectorizeUnlessFallback = false; public boolean skipVectorize = false; static { @@ -888,7 +890,9 @@ public void testQuery( protected QueryTestBuilder testBuilder() { return new QueryTestBuilder(new CalciteTestConfig()) - .cannotVectorize(cannotVectorize) + .cannotVectorize( + cannotVectorize || (!ExpressionProcessing.allowVectorizeFallback() && cannotVectorizeUnlessFallback) + ) .skipVectorize(skipVectorize); } @@ -1307,6 +1311,11 @@ protected void cannotVectorize() cannotVectorize = true; } + protected void cannotVectorizeUnlessFallback() + { + cannotVectorizeUnlessFallback = true; + } + protected void skipVectorize() { skipVectorize = true; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 531da0f8d66b..d98d5759b81b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -1528,6 +1528,7 @@ public void testArraySliceArrayColumns() @Test public void testArrayLength() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1, ARRAY_LENGTH(dim3), SUM(cnt) FROM druid.numfoo GROUP BY 1, 2 ORDER BY 2 DESC", ImmutableList.of( @@ -1740,6 +1741,7 @@ public void testArrayPrepend() @Test public void testArrayPrependAppend() { + cannotVectorizeUnlessFallback(); ImmutableList results; if (useDefault) { results = ImmutableList.of( @@ -1858,6 +1860,7 @@ public void testArrayConcat() @Test public void testArrayOffset() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT ARRAY_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2179,6 +2182,7 @@ public void testArrayGroupAsArrayWithFunction() @Test public void testArrayOrdinal() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT ARRAY_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2219,6 +2223,7 @@ public void testArrayOrdinal() @Test public void testArrayOffsetOf() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT ARRAY_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2265,6 +2270,7 @@ public void testArrayOffsetOf() @Test public void testArrayOrdinalOf() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT ARRAY_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -2312,6 +2318,7 @@ public void testArrayOrdinalOf() @Test public void testArrayToString() { + cannotVectorizeUnlessFallback(); ImmutableList results; if (useDefault) { results = ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 4ab78beb945e..fff819e68eee 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4223,6 +4223,7 @@ public void testSemiAndAntiJoinSimultaneouslyUsingExplicitJoins(Map results; if (useDefault) { results = ImmutableList.of( @@ -812,6 +814,7 @@ public void testMultiValueStringConcatBackwardsCompat0dot22andOlder() @Test public void testMultiValueStringOffset() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT MV_OFFSET(dim3, 1), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -848,6 +851,7 @@ public void testMultiValueStringOffset() @Test public void testMultiValueStringOrdinal() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT MV_ORDINAL(dim3, 2), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -888,6 +892,7 @@ public void testMultiValueStringOrdinal() @Test public void testMultiValueStringOffsetOf() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT MV_OFFSET_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -934,6 +939,7 @@ public void testMultiValueStringOffsetOf() @Test public void testMultiValueStringOrdinalOf() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT MV_ORDINAL_OF(dim3, 'b'), SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", ImmutableList.of( @@ -981,6 +987,7 @@ public void testMultiValueStringOrdinalOf() @Test public void testMultiValueStringToString() { + cannotVectorizeUnlessFallback(); ImmutableList results; if (useDefault) { results = ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 013753a02fcc..ec4d5812dd5d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -2683,6 +2683,7 @@ public void testGroupByPathSelectorFilter() @Test public void testGroupByPathSelectorFilterCoalesce() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT " + "JSON_VALUE(nest, '$.x'), " @@ -7629,6 +7630,7 @@ public void testGroupByAutoDouble() @Test public void testToJsonString() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT TO_JSON_STRING(nester) FROM druid.nested GROUP BY 1", ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 7a038b83b9ad..928139fd0140 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -589,6 +589,7 @@ public void testGroupByWithForceLimitPushDown() @Test public void testSafeDivide() { + cannotVectorizeUnlessFallback(); final Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); testQuery( @@ -2744,6 +2745,7 @@ public void testHavingOnExactCountDistinct() @Test public void testExactCountDistinctWithFilter() { + cannotVectorizeUnlessFallback(); msqIncompatible(); final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE foo.cnt = 1), SUM(foo.cnt) FROM druid.foo"; @@ -2770,6 +2772,9 @@ public void testExactCountDistinctWithFilter() @Test public void testExactCountDistinctWithFilter2() { + if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); + } final String sqlQuery = "SELECT COUNT(DISTINCT foo.dim1) FILTER(WHERE foo.cnt = 1), SUM(foo.cnt) FROM druid.foo"; testQuery( @@ -3259,6 +3264,7 @@ public void testPruneDeadAggregatorsThroughHaving() @Test public void testGroupByCaseWhen() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -3382,6 +3388,7 @@ public void testDecomposeCaseWhenTwoArg() @Test public void testGroupByCaseWhenOfTripleAnd() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " CASE WHEN m1 > 1 AND m1 < 5 AND cnt = 1 THEN 'x' ELSE NULL END," @@ -4493,6 +4500,9 @@ public void testCountStar() @Test public void testCountStarOnCommonTableExpression() { + if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); + } Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -4530,6 +4540,9 @@ public void testCountStarOnCommonTableExpression() @Test public void testCountStarOnView() { + if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); + } Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -4574,6 +4587,7 @@ public void testConfusedView() .aggregators(aggregators(new CountAggregatorFactory("a0"))) .context(QUERY_CONTEXT_DEFAULT); if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); builder = builder.virtualColumns( expressionVirtualColumn("v0", "substring(\"dim1\", 0, 1)", ColumnType.STRING) ) @@ -5156,6 +5170,7 @@ public void testGroupByWithSortOnPostAggregationNoTopNContext() @Test public void testFilteredAggregations() { + cannotVectorizeUnlessFallback(); Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -5340,6 +5355,7 @@ public void testFilteredAggregations() @Test public void testCaseFilteredAggregationWithGroupBy() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " cnt,\n" @@ -5419,6 +5435,7 @@ public void testFilteredAggregationWithNotIn() @Test public void testExpressionAggregations() { + cannotVectorizeUnlessFallback(); final ExprMacroTable macroTable = CalciteTests.createExprMacroTable(); testQuery( @@ -5849,6 +5866,7 @@ public void testInIsNotTrueAndLessThanFilter() @Test public void testInExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi'), COUNT(*)\n" + "FROM druid.foo\n" @@ -5912,6 +5930,7 @@ public void testInExpressionBelowThreshold() @Test public void testInOrIsNullExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL, COUNT(*)\n" + "FROM druid.foo\n" @@ -5943,6 +5962,7 @@ public void testInOrIsNullExpression() @Test public void testNotInOrIsNullExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT NOT (dim1 IN ('abc', 'def', 'ghi') OR dim1 IS NULL), COUNT(*)\n" + "FROM druid.foo\n" @@ -5974,6 +5994,7 @@ public void testNotInOrIsNullExpression() @Test public void testNotInAndIsNotNullExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 IS NOT NULL, COUNT(*)\n" + "FROM druid.foo\n" @@ -6005,6 +6026,7 @@ public void testNotInAndIsNotNullExpression() @Test public void testInOrGreaterThanExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 IN ('abc', 'def', 'ghi') OR dim1 > 'zzz', COUNT(*)\n" + "FROM druid.foo\n" @@ -6036,6 +6058,7 @@ public void testInOrGreaterThanExpression() @Test public void testNotInAndLessThanExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') AND dim1 < 'zzz', COUNT(*)\n" + "FROM druid.foo\n" @@ -6067,6 +6090,7 @@ public void testNotInAndLessThanExpression() @Test public void testNotInOrEqualToOneOfThemExpression() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT dim1 NOT IN ('abc', 'def', 'ghi') OR dim1 = 'def', COUNT(*)\n" + "FROM druid.foo\n" @@ -7333,6 +7357,7 @@ public void testSumOfString() @Test public void testSumOfExtractionFn() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT SUM(CAST(SUBSTRING(dim1, 1, 10) AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -8677,6 +8702,7 @@ public void testCountDistinctOfSubstring() @Test public void testCountDistinctOfTrim() { + cannotVectorizeUnlessFallback(); // Test a couple different syntax variants of TRIM. testQuery( "SELECT COUNT(DISTINCT TRIM(BOTH ' ' FROM dim1)) FROM druid.foo WHERE TRIM(dim1) <> ''", @@ -8710,6 +8736,7 @@ public void testCountDistinctOfTrim() @Test public void testSillyQuarters() { + cannotVectorizeUnlessFallback(); // Like FLOOR(__time TO QUARTER) but silly. testQuery( "SELECT CAST((EXTRACT(MONTH FROM __time) - 1 ) / 3 + 1 AS INTEGER) AS quarter, COUNT(*)\n" @@ -8820,6 +8847,7 @@ public void testRegexpExtractWithBadRegexPattern() @Test public void testRegexpExtractFilterViaNotNullCheck() { + cannotVectorizeUnlessFallback(); Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -9254,6 +9282,7 @@ public void testFilterOnTimeFloorComparisonMisaligned() @Test public void testFilterOnTimeExtract() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" @@ -9286,6 +9315,7 @@ public void testFilterOnTimeExtract() @Test public void testFilterOnTimeExtractWithMultipleDays() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" @@ -9326,6 +9356,7 @@ public void testFilterOnTimeExtractWithMultipleDays() @Test public void testFilterOnTimeExtractWithVariousTimeUnits() { + cannotVectorizeUnlessFallback(); msqIncompatible(); testQuery( "SELECT COUNT(*) FROM druid.foo4\n" @@ -9434,6 +9465,7 @@ public void testGroupByFloor() @Test public void testQueryWithSelectProjectAndIdentityProjectDoesNotRename() { + cannotVectorizeUnlessFallback(); msqIncompatible(); testQuery( PLANNER_CONFIG_NO_HLL.withOverrides( @@ -9711,6 +9743,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() @Test public void testGroupByStringLength() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1)", ImmutableList.of( @@ -10950,6 +10983,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() @Test public void testGroupByExtractYear() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " EXTRACT(YEAR FROM __time) AS \"year\",\n" @@ -10998,6 +11032,7 @@ public void testGroupByExtractYear() @Test public void testGroupByFormatYearAndMonth() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " TIME_FORMAt(__time, 'yyyy MM') AS \"year\",\n" @@ -11046,6 +11081,7 @@ public void testGroupByFormatYearAndMonth() @Test public void testGroupByExtractFloorTime() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + "EXTRACT(YEAR FROM FLOOR(__time TO YEAR)) AS \"year\", SUM(cnt)\n" @@ -11078,6 +11114,7 @@ public void testGroupByExtractFloorTime() @Test public void testGroupByExtractFloorTimeLosAngeles() { + cannotVectorizeUnlessFallback(); testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_LOS_ANGELES, @@ -14122,6 +14159,7 @@ public void testCountAndAverageByConstantVirtualColumn() @Test public void testExpressionCounts() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT\n" + " COUNT(reverse(dim2)),\n" @@ -15117,6 +15155,9 @@ public void testPlanWithInFilterLessThanInSubQueryThreshold() @Test public void testGreatestFunctionForNumberWithIsNull() { + if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); + } String query = "SELECT dim1, MAX(GREATEST(l1, l2)) IS NULL FROM druid.numfoo GROUP BY dim1"; List expectedResult; @@ -15182,6 +15223,7 @@ public void testGreatestFunctionForNumberWithIsNull() @Test public void testGreatestFunctionForStringWithIsNull() { + cannotVectorizeUnlessFallback(); msqIncompatible(); String query = "SELECT l1, LATEST(GREATEST(dim1, dim2)) IS NULL FROM druid.numfoo GROUP BY l1"; @@ -15365,6 +15407,7 @@ public void testComplexDecode() @Test public void testComplexDecodeAgg() { + cannotVectorizeUnlessFallback(); msqIncompatible(); testQuery( "SELECT APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1)))) from druid.foo", @@ -15398,6 +15441,7 @@ public void testComplexDecodeAgg() @Test public void testComplexDecodeAggWithCastedTypeName() { + cannotVectorizeUnlessFallback(); msqIncompatible(); testQuery( "SELECT " @@ -16434,6 +16478,7 @@ public void testStringOperationsNullableInference() @Test public void testGroupingSetsWithAggregateCase() { + cannotVectorizeUnlessFallback(); msqIncompatible(); final Map queryContext = ImmutableMap.of( PlannerConfig.CTX_KEY_USE_APPROXIMATE_COUNT_DISTINCT, false, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 02b49af58807..e053da4d7442 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -650,6 +650,9 @@ public void testSelectStarWithDimFilter() @Test public void testSelectDistinctWithCascadeExtractionFilter() { + if (NullHandling.sqlCompatible()) { + cannotVectorizeUnlessFallback(); + } testQuery( "SELECT distinct dim1 FROM druid.foo WHERE substring(substring(dim1, 2), 1, 1) = 'e' OR dim2 = 'a'", ImmutableList.of( @@ -694,6 +697,7 @@ public void testSelectDistinctWithCascadeExtractionFilter() @Test public void testSelectDistinctWithStrlenFilter() { + cannotVectorizeUnlessFallback(); testQuery( "SELECT distinct dim1 FROM druid.foo " + "WHERE CHARACTER_LENGTH(dim1) = 3 OR CAST(CHARACTER_LENGTH(dim1) AS varchar) = 3", @@ -2016,6 +2020,7 @@ public void testCountDistinctNonApproximateBasic() @Test public void testCountDistinctNonApproximateWithFilter() { + cannotVectorizeUnlessFallback(); testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( @@ -2054,6 +2059,7 @@ public void testCountDistinctNonApproximateWithFilter() @Test public void testCountDistinctNonApproximateWithFilterHaving() { + cannotVectorizeUnlessFallback(); testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java index 03d9357c7d48..b8d638e59ca0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java @@ -456,6 +456,7 @@ public void testGroupByWithPostAggregatorReferencingTimeFloorColumnOnTimeseries( if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { cannotVectorize(); } + cannotVectorizeUnlessFallback(); testQuery( "SELECT TIME_FORMAT(\"date\", 'yyyy-MM'), SUM(x)\n" + "FROM (\n" @@ -1611,6 +1612,7 @@ public void testTimeseriesSubqueryWithEarliestAggregator(String testName, Map queryContext) { + cannotVectorizeUnlessFallback(); DimFilter filter = NullHandling.replaceWithDefault() ? new InDimFilter("v0", new HashSet<>(Arrays.asList("1", "17"))) : new TypedInFilter("v0", ColumnType.LONG, null, ImmutableList.of(1, 17), null); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java index 8fde5d166b69..75b786d1f189 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java @@ -141,6 +141,7 @@ public void testMinTimeQueryWithTimeAndColumnFilters() @Test public void testMinTimeQueryWithTimeAndExpressionFilters() { + cannotVectorizeUnlessFallback(); HashMap queryContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); queryContext.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true); testQuery( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java index e6311f8d8345..3e89d7a17822 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; import org.apache.druid.quidem.DruidQTestInfo; @@ -54,10 +55,16 @@ public void beforeEach(ExtensionContext context) } private static final ImmutableMap CONTEXT_OVERRIDES = ImmutableMap.builder() - .putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT) - .put(PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE, PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED) - .put(QueryContexts.ENABLE_DEBUG, true) - .build(); + .putAll(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT) + .put( + PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE, + PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED + ) + .put( + QueryContexts.ENABLE_DEBUG, + true + ) + .build(); public QueryTestBuilder testBuilder() { @@ -97,7 +104,7 @@ public DruidQTestInfo getQTestInfo() qCaseDir, testName, "quidem testcase reason: " + decTestConfig.quidemReason() - ); + ); } else { return null; } @@ -115,10 +122,12 @@ public QueryTestBuilder expectedQueries(List> expectedQueries) return super.expectedQueries(expectedQueries); } } - } - .cannotVectorize(baseTest.cannotVectorize) - .skipVectorize(baseTest.skipVectorize); + }; - return builder; + return builder.cannotVectorize( + baseTest.cannotVectorize || + (!ExpressionProcessing.allowVectorizeFallback() && baseTest.cannotVectorizeUnlessFallback) + ) + .skipVectorize(baseTest.skipVectorize); } }