Allow non literal rhs in MV_FILTER_ONLY and MV_FILTER_NONE (apache#16113

)

This commit allows to use the MV_FILTER_ONLY & MV_FILTER_NONE functions
with a non literal argument. 
Currently `select mv_filter_only('mvd_dim', 'array_dim') from 'table'`
returns a `Unhandled Query Planning Failure`

This is being tackled and also considered for the cases where the `array_dim`
having null & empty values.

Changed classes:
 * `MultiValueStringOperatorConversions`
 * `ApplyFunction`
 * `CalciteMultiValueStringQueryTest`

processing/src/main/java/org/apache/druid/math/expr/ApplyFunction.java

@@ -442,11 +442,15 @@ public ExprEval apply(LambdaExpr lambdaExpr, List<Expr> argsExpr, Expr.ObjectBin
 
       Object[] array = arrayEval.asArray();
       if (array == null) {
-        return ExprEval.of(null);
+        return ExprEval.ofArray(arrayEval.asArrayType(), null);
       }
 
       SettableLambdaBinding lambdaBinding = new SettableLambdaBinding(arrayEval.elementType(), lambdaExpr, bindings);
       Object[] filtered = filter(arrayEval.asArray(), lambdaExpr, lambdaBinding).toArray();
+      // return null array expr if nothing is left in filtered
+      if (filtered.length == 0) {
+        return ExprEval.ofArray(arrayEval.asArrayType(), null);
+      }
       return ExprEval.ofArray(arrayEval.asArrayType(), filtered);
     }
 

processing/src/test/java/org/apache/druid/math/expr/ApplyFunctionTest.java

@@ -93,6 +93,10 @@ public void testFilter()
 
     assertExpr("filter((x) -> x > 2, [1, 2, 3, 4, 5])", new Long[] {3L, 4L, 5L});
     assertExpr("filter((x) -> x > 2, b)", new Long[] {3L, 4L, 5L});
+
+    String dummyNull = null;
+    assertExpr("filter((x) -> array_contains([], x), ['a', 'b'])", dummyNull);
+    assertExpr("filter((x) -> array_contains([], x), null)", dummyNull);
   }
 
   @Test

sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java

@@ -461,20 +461,6 @@ public DruidExpression toDruidExpression(
         return null;
       }
 
-      Expr expr = plannerContext.parseExpression(druidExpressions.get(1).getExpression());
-      // the right expression must be a literal array for this to work, since we need the values of the column
-      if (!expr.isLiteral()) {
-        return null;
-      }
-      Object[] lit = expr.eval(InputBindings.nilBindings()).asArray();
-      if (lit == null || lit.length == 0) {
-        return null;
-      }
-      HashSet<String> literals = Sets.newHashSetWithExpectedSize(lit.length);
-      for (Object o : lit) {
-        literals.add(Evals.asString(o));
-      }
-
       final DruidExpression.ExpressionGenerator builder = (args) -> {
         final StringBuilder expressionBuilder;
         if (isAllowList()) {
@@ -490,7 +476,17 @@ public DruidExpression toDruidExpression(
         return expressionBuilder.toString();
       };
 
-      if (druidExpressions.get(0).isSimpleExtraction()) {
+      Expr expr = plannerContext.parseExpression(druidExpressions.get(1).getExpression());
+      if (druidExpressions.get(0).isSimpleExtraction() && expr.isLiteral()) {
+        Object[] lit = expr.eval(InputBindings.nilBindings()).asArray();
+        if (lit == null || lit.length == 0) {
+          return null;
+        }
+        HashSet<String> literals = Sets.newHashSetWithExpectedSize(lit.length);
+        for (Object o : lit) {
+          literals.add(Evals.asString(o));
+        }
+
         DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
             Calcites.getColumnTypeForRelDataType(rexNode.getType()),
             builder,

sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java

@@ -1336,6 +1336,41 @@ public void testMultiValueListFilter()
     );
   }
 
+  @Test
+  public void testMultiValueListFilterNonLiteral()
+  {
+    // Cannot vectorize due to usage of expressions.
+    cannotVectorize();
+
+    testQuery(
+        "SELECT MV_FILTER_ONLY(dim3, ARRAY[dim2]) FROM druid.numfoo",
+        ImmutableList.of(
+            newScanQueryBuilder()
+                .dataSource(CalciteTests.DATASOURCE3)
+                .intervals(querySegmentSpec(Filtration.eternity()))
+                .virtualColumns(
+                    new ExpressionVirtualColumn(
+                        "v0",
+                        "filter((x) -> array_contains(array(\"dim2\"), x), \"dim3\")",
+                        ColumnType.STRING,
+                        TestExprMacroTable.INSTANCE
+                    )
+                )
+                .columns("v0")
+                .context(QUERY_CONTEXT_DEFAULT)
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{"a"},
+            new Object[]{NullHandling.defaultStringValue()},
+            new Object[]{NullHandling.defaultStringValue()},
+            new Object[]{NullHandling.defaultStringValue()},
+            new Object[]{NullHandling.defaultStringValue()},
+            new Object[]{NullHandling.defaultStringValue()}
+        )
+    );
+  }
+
   @Test
   public void testMultiValueListFilterDeny()
   {
@@ -1391,6 +1426,41 @@ public void testMultiValueListFilterDeny()
     );
   }
 
+  @Test
+  public void testMultiValueListFilterDenyNonLiteral()
+  {
+    // Cannot vectorize due to usage of expressions.
+    cannotVectorize();
+
+    testQuery(
+        "SELECT MV_FILTER_NONE(dim3, ARRAY[dim2]) FROM druid.numfoo",
+        ImmutableList.of(
+            newScanQueryBuilder()
+                .dataSource(CalciteTests.DATASOURCE3)
+                .intervals(querySegmentSpec(Filtration.eternity()))
+                .virtualColumns(
+                    new ExpressionVirtualColumn(
+                        "v0",
+                        "filter((x) -> !array_contains(array(\"dim2\"), x), \"dim3\")",
+                        ColumnType.STRING,
+                        TestExprMacroTable.INSTANCE
+                    )
+                )
+                .columns("v0")
+                .context(QUERY_CONTEXT_DEFAULT)
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{"b"},
+            new Object[]{"[\"b\",\"c\"]"},
+            new Object[]{"d"},
+            new Object[]{""},
+            new Object[]{NullHandling.defaultStringValue()},
+            new Object[]{NullHandling.defaultStringValue()}
+        )
+    );
+  }
+
   @Test
   public void testMultiValueListFilterComposed()
   {