expression vector processing improvements

changes: * introduces FilteredInputBinding which adds better conditional expression processing support using a VectorMatch internally to selectively evaluate input vectors instead of precomputing all inputs, with nvl updated to take advantage of this * refactor some stuff to streamline expression vector processor implementation for simple functions like most math and logical operations with some new factory classes * update vector identifier expression processor to delegate evaluating results directly to the input binding selectors with ExprEvalBindingVector * add maxVectorSize() to ExprVectorProcessor to avoid having to pass max vector size around everywhere
apache · Dec 12, 2024 · 405d21d · 405d21d
1 parent 3c1b488
commit 405d21d
Show file tree

Hide file tree

Showing 90 changed files with 4,208 additions and 3,854 deletions.
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
@@ -139,7 +139,23 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark
       "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM expressions",
       // 46: filters with random orders
       "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND string1 = '1000' GROUP BY 1 ORDER BY 2",
-      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2"
+      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2",
+      // 48-57 nvl tests
+      // lower cardinality
+      "SELECT NVL(string2, string1), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string2, CONCAT(string1, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // higher cardinality
+      "SELECT NVL(string5, string3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string5, CONCAT(string3, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // no lhs nulls
+      "SELECT NVL(string1, string3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string1, CONCAT(string3, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // numeric
+      "SELECT NVL(long5, long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(long5, long1 + long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // numeric no lhs null
+      "SELECT NVL(long1, long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(long1, long5 + long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2"
   );
 
   @Param({
@@ -200,7 +216,17 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark
       "44",
       "45",
       "46",
-      "47"
+      "47",
+      "48",
+      "49",
+      "50",
+      "51",
+      "52",
+      "53",
+      "54",
+      "55",
+      "56",
+      "57"
   })
   private String query;
 

diff --git a/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java
@@ -80,7 +80,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.lessThan(inspector, left, right);
+    return VectorComparisonProcessors.lessThan().asProcessor(inspector, left, right);
   }
 }
 
@@ -128,7 +128,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.lessThanOrEqual(inspector, left, right);
+    return VectorComparisonProcessors.lessThanOrEquals().asProcessor(inspector, left, right);
   }
 }
 
@@ -176,7 +176,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.greaterThan(inspector, left, right);
+    return VectorComparisonProcessors.greaterThan().asProcessor(inspector, left, right);
   }
 }
 
@@ -224,7 +224,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.greaterThanOrEqual(inspector, left, right);
+    return VectorComparisonProcessors.greaterThanOrEquals().asProcessor(inspector, left, right);
   }
 }
 
@@ -271,7 +271,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.equal(inspector, left, right);
+    return VectorComparisonProcessors.equals().asProcessor(inspector, left, right);
   }
 }
 
@@ -318,7 +318,7 @@ protected final boolean evalDouble(double left, double right)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorComparisonProcessors.notEqual(inspector, left, right);
+    return VectorComparisonProcessors.notEquals().asProcessor(inspector, left, right);
   }
 }
 

diff --git a/processing/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java b/processing/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java
@@ -85,7 +85,7 @@ public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector
     if (Types.is(type, ExprType.STRING)) {
       return VectorStringProcessors.concat(inspector, left, right);
     }
-    return VectorMathProcessors.plus(inspector, left, right);
+    return VectorMathProcessors.plus().asProcessor(inspector, left, right);
   }
 }
 
@@ -124,7 +124,7 @@ public boolean canVectorize(InputBindingInspector inspector)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorMathProcessors.minus(inspector, left, right);
+    return VectorMathProcessors.subtract().asProcessor(inspector, left, right);
   }
 }
 
@@ -163,7 +163,7 @@ public boolean canVectorize(InputBindingInspector inspector)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorMathProcessors.multiply(inspector, left, right);
+    return VectorMathProcessors.multiply().asProcessor(inspector, left, right);
   }
 }
 
@@ -202,7 +202,7 @@ public boolean canVectorize(InputBindingInspector inspector)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorMathProcessors.divide(inspector, left, right);
+    return VectorMathProcessors.divide().asProcessor(inspector, left, right);
   }
 }
 
@@ -241,7 +241,7 @@ public boolean canVectorize(InputBindingInspector inspector)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorMathProcessors.power(inspector, left, right);
+    return VectorMathProcessors.power().asProcessor(inspector, left, right);
   }
 }
 
@@ -280,6 +280,6 @@ public boolean canVectorize(InputBindingInspector inspector)
   @Override
   public <T> ExprVectorProcessor<T> asVectorProcessor(VectorInputBindingInspector inspector)
   {
-    return VectorMathProcessors.modulo(inspector, left, right);
+    return VectorMathProcessors.modulo().asProcessor(inspector, left, right);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Expr.java b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
@@ -468,7 +468,7 @@ interface ObjectBinding extends InputBindingInspector
    */
   interface VectorInputBinding extends VectorInputBindingInspector
   {
-    <T> T[] getObjectVector(String name);
+    Object[] getObjectVector(String name);
 
     long[] getLongVector(String name);
 

diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionValidationException.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionValidationException.java
@@ -29,8 +29,18 @@ public ExpressionValidationException(NamedFunction fn, String msg, Object... for
     super("Function[%s] %s", fn.name(), StringUtils.format(msg, formatArgs));
   }
 
+  public ExpressionValidationException(String name, String msg, Object... formatArgs)
+  {
+    super("Function[%s] %s", name, StringUtils.format(msg, formatArgs));
+  }
+
   public ExpressionValidationException(NamedFunction fn, Throwable e, String msg, Object... formatArgs)
   {
     super(e, "Function[%s] %s", fn.name(), StringUtils.format(msg, formatArgs));
   }
+
+  public ExpressionValidationException(String name, Throwable e, String msg, Object... formatArgs)
+  {
+    super(e, "Function[%s] %s", name, StringUtils.format(msg, formatArgs));
+  }
 }