Allow aliasing of Macros and add new alias for complex decode 64 (apa…

…che#15034)

* Add AliasExprMacro to allow aliasing of native expression macros
* Add decode_base64_complex alias for complex_decode_base64

extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java

@@ -251,7 +251,7 @@ public void testMetricsSumEstimateIntersect()
                        + "  SUM(cnt),\n"
                        + "  DS_TUPLE_DOUBLES_METRICS_SUM_ESTIMATE(DS_TUPLE_DOUBLES(tuplesketch_dim2)) AS all_sum_estimates,\n"
                        + StringUtils.replace(
-        "DS_TUPLE_DOUBLES_METRICS_SUM_ESTIMATE(DS_TUPLE_DOUBLES_INTERSECT(COMPLEX_DECODE_BASE64('arrayOfDoublesSketch', '%s'), DS_TUPLE_DOUBLES(tuplesketch_dim2), 128)) AS intersect_sum_estimates\n",
+        "DS_TUPLE_DOUBLES_METRICS_SUM_ESTIMATE(DS_TUPLE_DOUBLES_INTERSECT(DECODE_BASE64_COMPLEX('arrayOfDoublesSketch', '%s'), DS_TUPLE_DOUBLES(tuplesketch_dim2), 128)) AS intersect_sum_estimates\n",
         "%s",
         COMPACT_BASE_64_ENCODED_SKETCH_FOR_INTERSECTION
     )

processing/src/main/java/org/apache/druid/math/expr/BuiltInExprMacros.java

@@ -32,7 +32,13 @@ public class BuiltInExprMacros
   public static class ComplexDecodeBase64ExprMacro implements ExprMacroTable.ExprMacro
   {
     public static final String NAME = "complex_decode_base64";
+    public static final String ALIAS = "decode_base64_complex";
 
+    /**
+     * use name() in closure scope to allow Alias macro to override it with alias.
+     *
+     * @return String
+     */
     @Override
     public String name()
     {
@@ -52,7 +58,7 @@ final class ComplexDecodeBase64Expression extends ExprMacroTable.BaseScalarMacro
 
       public ComplexDecodeBase64Expression(List<Expr> args)
       {
-        super(NAME, args);
+        super(name(), args);
         validationHelperCheckArgumentCount(args, 2);
         final Expr arg0 = args.get(0);
 
@@ -148,7 +154,6 @@ public Object getLiteralValue()
 
   public static class StringDecodeBase64UTFExprMacro implements ExprMacroTable.ExprMacro
   {
-
     public static final String NAME = "decode_base64_utf8";
 
     @Override
@@ -158,6 +163,11 @@ public Expr apply(List<Expr> args)
       return new StringDecodeBase64UTFExpression(args.get(0));
     }
 
+    /**
+     * use name() in closure scope to allow Alias macro to override it with alias.
+     *
+     * @return String
+     */
     @Override
     public String name()
     {
@@ -168,7 +178,7 @@ final class StringDecodeBase64UTFExpression extends ExprMacroTable.BaseScalarUni
     {
       public StringDecodeBase64UTFExpression(Expr arg)
       {
-        super(NAME, arg);
+        super(name(), arg);
       }
 
       @Override
@@ -214,4 +224,5 @@ public Object getLiteralValue()
       }
     }
   }
+
 }

processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java

@@ -43,8 +43,13 @@
  */
 public class ExprMacroTable
 {
+  private static final BuiltInExprMacros.ComplexDecodeBase64ExprMacro COMPLEX_DECODE_BASE_64_EXPR_MACRO = new BuiltInExprMacros.ComplexDecodeBase64ExprMacro();
   private static final List<ExprMacro> BUILT_IN = ImmutableList.of(
-      new BuiltInExprMacros.ComplexDecodeBase64ExprMacro(),
+      COMPLEX_DECODE_BASE_64_EXPR_MACRO,
+      new AliasExprMacro(
+          COMPLEX_DECODE_BASE_64_EXPR_MACRO,
+          BuiltInExprMacros.ComplexDecodeBase64ExprMacro.ALIAS
+      ),
       new BuiltInExprMacros.StringDecodeBase64UTFExprMacro()
   );
   private static final ExprMacroTable NIL = new ExprMacroTable(Collections.emptyList());
@@ -247,4 +252,32 @@ public String toString()
       return StringUtils.format("(%s %s)", name, getArgs());
     }
   }
+
+  /***
+   * Alias Expression macro to create an alias and delegate operations to the same base macro.
+   * The Expr spit out by the apply method should use name() in all the places instead of an internal constant so that things like error messages behave correctly.
+   */
+  static class AliasExprMacro implements ExprMacroTable.ExprMacro
+  {
+    private final ExprMacroTable.ExprMacro exprMacro;
+    private final String alias;
+
+    public AliasExprMacro(final ExprMacroTable.ExprMacro baseExprMacro, final String alias)
+    {
+      this.exprMacro = baseExprMacro;
+      this.alias = alias;
+    }
+
+    @Override
+    public Expr apply(List<Expr> args)
+    {
+      return exprMacro.apply(args);
+    }
+
+    @Override
+    public String name()
+    {
+      return alias;
+    }
+  }
 }

processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java

@@ -41,6 +41,9 @@
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 
 public class FunctionTest extends InitializedNullHandlingTest
@@ -952,8 +955,47 @@ public void testComplexDecode()
         ),
         expected
     );
+    // test with alias
+    assertExpr(
+        StringUtils.format(
+            "decode_base64_complex('%s', '%s')",
+            TypeStrategiesTest.NULLABLE_TEST_PAIR_TYPE.getComplexTypeName(),
+            StringUtils.encodeBase64String(bytes)
+        ),
+        expected
+    );
   }
 
+  @Test
+  public void testMacrosWithMultipleAliases()
+  {
+    ExprMacroTable.ExprMacro drinkMacro = new ExprMacroTable.ExprMacro()
+    {
+      @Override
+      public Expr apply(List<Expr> args)
+      {
+        return new StringExpr("happiness");
+      }
+
+      @Override
+      public String name()
+      {
+        return "drink";
+      }
+    };
+    List<ExprMacroTable.ExprMacro> macros = new ArrayList<>();
+    macros.add(drinkMacro);
+    List<String> aliases = Arrays.asList("tea", "coffee", "chai", "chaha", "kevha", "chay");
+    for (String tea : aliases) {
+      macros.add(new ExprMacroTable.AliasExprMacro(drinkMacro, tea));
+    }
+    final ExprMacroTable exprMacroTable = new ExprMacroTable(macros);
+    final Expr happiness = new StringExpr("happiness");
+    Assert.assertEquals(happiness, Parser.parse("drink(1,2)", exprMacroTable));
+    for (String tea : aliases) {
+      Assert.assertEquals(happiness, Parser.parse(StringUtils.format("%s(1,2)", tea), exprMacroTable));
+    }
+  }
   @Test
   public void testComplexDecodeNull()
   {
@@ -964,6 +1006,13 @@ public void testComplexDecodeNull()
         ),
         null
     );
+    assertExpr(
+        StringUtils.format(
+            "decode_base64_complex('%s', null)",
+            TypeStrategiesTest.NULLABLE_TEST_PAIR_TYPE.getComplexTypeName()
+        ),
+        null
+    );
   }
 
   @Test

sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java

@@ -32,6 +32,7 @@
 import org.apache.calcite.sql.validate.SqlNameMatcher;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.BuiltInExprMacros;
 import org.apache.druid.sql.calcite.aggregation.SqlAggregator;
 import org.apache.druid.sql.calcite.aggregation.builtin.ArrayConcatSqlAggregator;
 import org.apache.druid.sql.calcite.aggregation.builtin.ArraySqlAggregator;
@@ -227,11 +228,16 @@ public class DruidOperatorTable implements SqlOperatorTable
                    .add(ContainsOperatorConversion.caseInsensitive())
                    .build();
 
+  private static final SqlOperatorConversion COMPLEX_DECODE_OPERATOR_CONVERSIONS = new ComplexDecodeBase64OperatorConversion();
   private static final List<SqlOperatorConversion> VALUE_COERCION_OPERATOR_CONVERSIONS =
       ImmutableList.<SqlOperatorConversion>builder()
                    .add(new CastOperatorConversion())
                    .add(new ReinterpretOperatorConversion())
-                   .add(new ComplexDecodeBase64OperatorConversion())
+                   .add(COMPLEX_DECODE_OPERATOR_CONVERSIONS)
+                   .add(new AliasedOperatorConversion(
+                       COMPLEX_DECODE_OPERATOR_CONVERSIONS,
+                       BuiltInExprMacros.ComplexDecodeBase64ExprMacro.ALIAS
+                   ))
                    .add(new DecodeBase64UTFOperatorConversion())
                    .build();
 

sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java

@@ -30,6 +30,7 @@
 import org.apache.druid.java.util.common.HumanReadableBytes;
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.JodaUtils;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.PeriodGranularity;
@@ -14424,36 +14425,40 @@ public void testTimeseriesQueryWithEmptyInlineDatasourceAndGranularity()
   public void testComplexDecode()
   {
     cannotVectorize();
-    testQuery(
-        "SELECT COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1))) from druid.foo LIMIT 10",
-        ImmutableList.of(
-            Druids.newScanQueryBuilder()
-                  .dataSource(CalciteTests.DATASOURCE1)
-                  .intervals(querySegmentSpec(Filtration.eternity()))
-                  .columns("v0")
-                  .virtualColumns(
-                      expressionVirtualColumn(
-                          "v0",
-                          "complex_decode_base64('hyperUnique',parse_json(to_json_string(\"unique_dim1\")))",
-                          ColumnType.ofComplex("hyperUnique")
-                      )
-                  )
-                  .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
-                  .legacy(false)
-                  .limit(10)
-                  .build()
-        ),
-        ImmutableList.of(
-            new Object[]{"\"AQAAAEAAAA==\""},
-            new Object[]{"\"AQAAAQAAAAHNBA==\""},
-            new Object[]{"\"AQAAAQAAAAOzAg==\""},
-            new Object[]{"\"AQAAAQAAAAFREA==\""},
-            new Object[]{"\"AQAAAQAAAACyEA==\""},
-            new Object[]{"\"AQAAAQAAAAEkAQ==\""}
-        )
-    );
+    for (String complexDecode : Arrays.asList("COMPLEX_DECODE_BASE64", "DECODE_BASE64_COMPLEX")) {
+      testQuery(
+          StringUtils.format(
+              "SELECT %s('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1))) from druid.foo LIMIT 10",
+              complexDecode
+          ),
+          ImmutableList.of(
+              Druids.newScanQueryBuilder()
+                    .dataSource(CalciteTests.DATASOURCE1)
+                    .intervals(querySegmentSpec(Filtration.eternity()))
+                    .columns("v0")
+                    .virtualColumns(
+                        expressionVirtualColumn(
+                            "v0",
+                            "complex_decode_base64('hyperUnique',parse_json(to_json_string(\"unique_dim1\")))",
+                            ColumnType.ofComplex("hyperUnique")
+                        )
+                    )
+                    .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+                    .legacy(false)
+                    .limit(10)
+                    .build()
+          ),
+          ImmutableList.of(
+              new Object[]{"\"AQAAAEAAAA==\""},
+              new Object[]{"\"AQAAAQAAAAHNBA==\""},
+              new Object[]{"\"AQAAAQAAAAOzAg==\""},
+              new Object[]{"\"AQAAAQAAAAFREA==\""},
+              new Object[]{"\"AQAAAQAAAACyEA==\""},
+              new Object[]{"\"AQAAAQAAAAEkAQ==\""}
+          )
+      );
+    }
   }
-
   @Test
   public void testComplexDecodeAgg()
   {