Merge remote-tracking branch 'origin/master' into batch-upload-segments

pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTable.java

@@ -35,8 +35,6 @@
  * Data table is used to transfer data from server to broker.
  */
 public interface DataTable {
-  // TODO: remove this when we stop supporting DataTable V2.
-  String EXCEPTION_METADATA_KEY = "Exception";
 
   void addException(ProcessingException processingException);
 

pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java

@@ -106,7 +106,22 @@ public enum TransformFunctionType {
   // object type
   ARRAY_TO_MV("arrayToMV",
       ReturnTypes.cascade(opBinding -> positionalComponentReturnType(opBinding, 0), SqlTypeTransforms.FORCE_NULLABLE),
-      OperandTypes.family(SqlTypeFamily.ARRAY), "array_to_mv"),
+      OperandTypes.family(SqlTypeFamily.ARRAY), "array_to_mv") {
+
+    @Override
+    public boolean isDeterministic() {
+      // ARRAY_TO_MV is not deterministic. In fact, it has no implementation
+      // We need to explicitly set it as not deterministic in order to do not let Calcite to optimize expressions like
+      // `ARRAY_TO_MV(RandomAirports) = 'MFR' and ARRAY_TO_MV(RandomAirports) = 'GTR'` as `false`.
+      // If the function were deterministic, its value would never be MFR and GTR at the same time, so Calcite is
+      // smart enough to know there is no value that satisfies the condition.
+      // In fact what ARRAY_TO_MV does is just to trick Calcite typesystem, but then what the leaf stage executor
+      // receives is `RandomAirports = 'MFR' and RandomAirports = 'GTR'`, which in the V1 semantics means:
+      // true if and only if RandomAirports contains a value equal to 'MFR' and RandomAirports contains a value equal
+      // to 'GTR'
+      return false;
+    }
+  },
 
   // string functions
   JSON_EXTRACT_SCALAR("jsonExtractScalar",
@@ -358,6 +373,10 @@ public SqlFunctionCategory getSqlFunctionCategory() {
     return _sqlFunctionCategory;
   }
 
+  public boolean isDeterministic() {
+    return true;
+  }
+
   /** Returns the optional explicit returning type specification. */
   private static RelDataType positionalReturnTypeInferenceFromStringLiteral(SqlOperatorBinding opBinding, int pos) {
     return positionalReturnTypeInferenceFromStringLiteral(opBinding, pos, SqlTypeName.ANY);

pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java

@@ -305,4 +305,48 @@ public static Object arrayValueConstructor(Object... arr) {
     }
     return arr;
   }
+
+  @ScalarFunction
+  public static int[] generateIntArray(int start, int end, int inc) {
+    int size = (end - start) / inc + 1;
+    int[] arr = new int[size];
+
+    for (int i = 0, value = start; i < size; i++, value += inc) {
+      arr[i] = value;
+    }
+    return arr;
+  }
+
+  @ScalarFunction
+  public static long[] generateLongArray(long start, long end, long inc) {
+    int size = (int) ((end - start) / inc + 1);
+    long[] arr = new long[size];
+
+    for (int i = 0; i < size; i++, start += inc) {
+      arr[i] = start;
+    }
+    return arr;
+  }
+
+  @ScalarFunction
+  public static float[] generateFloatArray(float start, float end, float inc) {
+    int size = (int) ((end - start) / inc + 1);
+    float[] arr = new float[size];
+
+    for (int i = 0; i < size; i++, start += inc) {
+      arr[i] = start;
+    }
+    return arr;
+  }
+
+  @ScalarFunction
+  public static double[] generateDoubleArray(double start, double end, double inc) {
+    int size = (int) ((end - start) / inc + 1);
+    double[] arr = new double[size];
+
+    for (int i = 0; i < size; i++, start += inc) {
+      arr[i] = start;
+    }
+    return arr;
+  }
 }