SQL: Support GROUP BY and ORDER BY for NULL types.

extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java

@@ -386,16 +386,16 @@ public void testNullInputs()
                       ImmutableList.of(
                           new ArrayOfDoublesSketchToMetricsSumEstimatePostAggregator(
                               "p1",
-                              expressionPostAgg("p0", "null", null)
+                              expressionPostAgg("p0", "null", ColumnType.STRING)
                           ),
                           new ArrayOfDoublesSketchSetOpPostAggregator(
                               "p4",
                               ArrayOfDoublesSketchOperations.Operation.UNION.name(),
                               null,
                               null,
                               ImmutableList.of(
-                                  expressionPostAgg("p2", "null", null),
-                                  expressionPostAgg("p3", "null", null)
+                                  expressionPostAgg("p2", "null", ColumnType.STRING),
+                                  expressionPostAgg("p3", "null", ColumnType.STRING)
                               )
                           ),
                           new ArrayOfDoublesSketchSetOpPostAggregator(
@@ -404,7 +404,7 @@ public void testNullInputs()
                               null,
                               null,
                               ImmutableList.of(
-                                  expressionPostAgg("p5", "null", null),
+                                  expressionPostAgg("p5", "null", ColumnType.STRING),
                                   new FieldAccessPostAggregator("p6", "a1")
                               )
                           ),
@@ -415,7 +415,7 @@ public void testNullInputs()
                               null,
                               ImmutableList.of(
                                   new FieldAccessPostAggregator("p8", "a1"),
-                                  expressionPostAgg("p9", "null", null)
+                                  expressionPostAgg("p9", "null", ColumnType.STRING)
                               )
                           )
                       )

sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java

@@ -729,6 +729,11 @@ private static DimFilter toSimpleLeafFilter(
 
         // Numeric lhs needs a numeric comparison.
         final StringComparator comparator = Calcites.getStringComparatorForRelDataType(lhs.getType());
+        if (comparator == null) {
+          // Type is not comparable.
+          return null;
+        }
+
         final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator);
         final DimFilter filter;
 

sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java

@@ -206,7 +206,7 @@ public static ColumnType getValueTypeForRelDataTypeFull(final RelDataType type)
       return ColumnType.DOUBLE;
     } else if (isLongType(sqlTypeName)) {
       return ColumnType.LONG;
-    } else if (isStringType(sqlTypeName)) {
+    } else if (isStringType(sqlTypeName) || sqlTypeName == SqlTypeName.NULL) {
       return ColumnType.STRING;
     } else if (SqlTypeName.OTHER == sqlTypeName) {
       if (type instanceof RowSignatures.ComplexSqlType) {
@@ -244,12 +244,22 @@ public static boolean isLongType(SqlTypeName sqlTypeName)
   }
 
   /**
-   * Returns the natural StringComparator associated with the RelDataType
+   * Returns the natural StringComparator associated with the RelDataType, or null if the type is not convertible to
+   * {@link ColumnType} by {@link #getColumnTypeForRelDataType(RelDataType)}.
    */
+  @Nullable
   public static StringComparator getStringComparatorForRelDataType(RelDataType dataType)
   {
-    final ColumnType valueType = getColumnTypeForRelDataType(dataType);
-    return getStringComparatorForValueType(valueType);
+    if (dataType.getSqlTypeName() == SqlTypeName.NULL) {
+      return StringComparators.NATURAL;
+    } else {
+      final ColumnType valueType = getColumnTypeForRelDataType(dataType);
+      if (valueType == null) {
+        return null;
+      }
+
+      return getStringComparatorForValueType(valueType);
+    }
   }
 
   /**

sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java

@@ -6108,6 +6108,102 @@ public void testCountStarWithTimeInIntervalFilterLosAngeles()
     );
   }
 
+  @Test
+  public void testGroupByNullType()
+  {
+    // Cannot vectorize due to null constant expression.
+    cannotVectorize();
+    testQuery(
+        "SELECT NULL as nullcol, COUNT(*) FROM druid.foo GROUP BY 1",
+        ImmutableList.of(
+            new GroupByQuery.Builder()
+                .setDataSource(CalciteTests.DATASOURCE1)
+                .setInterval(querySegmentSpec(Filtration.eternity()))
+                .setGranularity(Granularities.ALL)
+                .setVirtualColumns(expressionVirtualColumn("v0", "null", ColumnType.STRING))
+                .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
+                .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0")))
+                .setContext(QUERY_CONTEXT_DEFAULT)
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{null, 6L}
+        )
+    );
+  }
+
+  @Test
+  public void testOrderByNullType()
+  {
+    // Cannot vectorize due to null constant expression.
+    cannotVectorize();
+    testQuery(
+        // Order on subquery, since the native engine doesn't currently support ordering when selecting directly
+        // from a table.
+        "SELECT dim1, NULL as nullcol FROM (SELECT DISTINCT dim1 FROM druid.foo LIMIT 1) ORDER BY 2",
+        ImmutableList.of(
+            WindowOperatorQueryBuilder
+                .builder()
+                .setDataSource(
+                    new TopNQueryBuilder()
+                        .dataSource(CalciteTests.DATASOURCE1)
+                        .intervals(querySegmentSpec(Filtration.eternity()))
+                        .dimension(new DefaultDimensionSpec("dim1", "d0", ColumnType.STRING))
+                        .threshold(1)
+                        .metric(new DimensionTopNMetricSpec(null, StringComparators.LEXICOGRAPHIC))
+                        .postAggregators(expressionPostAgg("s0", "null", ColumnType.STRING))
+                        .context(QUERY_CONTEXT_DEFAULT)
+                        .build()
+                )
+                .setSignature(
+                    RowSignature.builder()
+                                .add("d0", ColumnType.STRING)
+                                .add("s0", ColumnType.STRING)
+                                .build()
+                )
+                .setOperators(
+                    OperatorFactoryBuilders.naiveSortOperator("s0", ColumnWithDirection.Direction.ASC)
+                )
+                .setLeafOperators(
+                    OperatorFactoryBuilders
+                        .scanOperatorFactoryBuilder()
+                        .setOffsetLimit(0, Long.MAX_VALUE)
+                        .setProjectedColumns("d0", "s0")
+                        .build()
+                )
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{"", null}
+        )
+    );
+  }
+
+  @Test
+  public void testGroupByOrderByNullType()
+  {
+    // Cannot vectorize due to null constant expression.
+    cannotVectorize();
+
+    testQuery(
+        "SELECT NULL as nullcol, COUNT(*) FROM druid.foo GROUP BY 1 ORDER BY 1",
+        ImmutableList.of(
+            new GroupByQuery.Builder()
+                .setDataSource(CalciteTests.DATASOURCE1)
+                .setInterval(querySegmentSpec(Filtration.eternity()))
+                .setGranularity(Granularities.ALL)
+                .setVirtualColumns(expressionVirtualColumn("v0", "null", ColumnType.STRING))
+                .setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
+                .setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0")))
+                .setContext(QUERY_CONTEXT_DEFAULT)
+                .build()
+        ),
+        ImmutableList.of(
+            new Object[]{null, 6L}
+        )
+    );
+  }
+
   @Test
   public void testCountStarWithTimeInIntervalFilterInvalidInterval()
   {

sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java

@@ -141,7 +141,7 @@ public void testValuesContainingNull()
                         ImmutableList.of(new Object[]{null, "United States"}),
                         RowSignature
                             .builder()
-                            .add("EXPR$0", null)
+                            .add("EXPR$0", ColumnType.STRING)
                             .add("EXPR$1", ColumnType.STRING)
                             .build()
                     )

sql/src/test/java/org/apache/druid/sql/calcite/CalciteTableAppendTest.java

@@ -46,9 +46,7 @@ public void testUnion()
                     .columns("dim1", "v0")
                     .context(QUERY_CONTEXT_DEFAULT)
                     .resultFormat(ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
-                    .virtualColumns(
-                        expressionVirtualColumn("v0", "null", null)
-                    )
+                    .virtualColumns(expressionVirtualColumn("v0", "null", ColumnType.STRING))
                     .legacy(false)
                     .build(),
                 Druids.newScanQueryBuilder()

sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteQueryTest.java

@@ -26,6 +26,8 @@
 import org.apache.druid.sql.calcite.planner.PlannerConfig;
 import org.apache.druid.sql.calcite.util.SqlTestFramework;
 import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 @ExtendWith(NotYetSupportedProcessor.class)
@@ -64,4 +66,12 @@ public SqlTestFramework.PlannerFixture plannerFixture(PlannerConfig plannerConfi
 
     return builder;
   }
+
+  @Override
+  @Test
+  @Disabled("Unhandled Query Planning Failure")
+  public void testOrderByNullType()
+  {
+    super.testOrderByNullType();
+  }
 }

sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitesTest.java

@@ -20,52 +20,141 @@
 package org.apache.druid.sql.calcite.planner;
 
 import com.google.common.collect.ImmutableSortedSet;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.druid.query.ordering.StringComparators;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.sql.calcite.table.RowSignatures;
 import org.apache.druid.sql.calcite.util.CalciteTestBase;
-import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 public class CalcitesTest extends CalciteTestBase
 {
   @Test
   public void testEscapeStringLiteral()
   {
-    Assert.assertEquals("''", Calcites.escapeStringLiteral(""));
-    Assert.assertEquals("'foo'", Calcites.escapeStringLiteral("foo"));
-    Assert.assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar"));
-    Assert.assertEquals("U&'foö bar'", Calcites.escapeStringLiteral("foö bar"));
-    Assert.assertEquals("U&'foo \\0026\\0026 bar'", Calcites.escapeStringLiteral("foo && bar"));
-    Assert.assertEquals("U&'foo \\005C bar'", Calcites.escapeStringLiteral("foo \\ bar"));
-    Assert.assertEquals("U&'foo\\0027s bar'", Calcites.escapeStringLiteral("foo's bar"));
-    Assert.assertEquals("U&'друид'", Calcites.escapeStringLiteral("друид"));
+    assertEquals("''", Calcites.escapeStringLiteral(""));
+    assertEquals("'foo'", Calcites.escapeStringLiteral("foo"));
+    assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar"));
+    assertEquals("U&'foö bar'", Calcites.escapeStringLiteral("foö bar"));
+    assertEquals("U&'foo \\0026\\0026 bar'", Calcites.escapeStringLiteral("foo && bar"));
+    assertEquals("U&'foo \\005C bar'", Calcites.escapeStringLiteral("foo \\ bar"));
+    assertEquals("U&'foo\\0027s bar'", Calcites.escapeStringLiteral("foo's bar"));
+    assertEquals("U&'друид'", Calcites.escapeStringLiteral("друид"));
   }
 
   @Test
   public void testFindUnusedPrefix()
   {
-    Assert.assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar")));
-    Assert.assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x")));
-    Assert.assertEquals("_x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x0")));
-    Assert.assertEquals("_x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x4")));
-    Assert.assertEquals("__x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x2xx", "x0")));
-    Assert.assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x2xx", " x")));
-    Assert.assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "_xbxx")));
-    Assert.assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x")));
-    Assert.assertEquals("__x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "x1a", "_x90")));
+    assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar")));
+    assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x")));
+    assertEquals("_x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x0")));
+    assertEquals("_x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "bar", "x4")));
+    assertEquals("__x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x2xx", "x0")));
+    assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x2xx", " x")));
+    assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "_xbxx")));
+    assertEquals("x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "xa", "_x")));
+    assertEquals("__x", Calcites.findUnusedPrefixForDigits("x", ImmutableSortedSet.of("foo", "x1a", "_x90")));
+  }
+
+  @Test
+  public void testGetStringComparatorForRelDataType()
+  {
+    for (final SqlTypeName typeName : SqlTypeFamily.CHARACTER.getTypeNames()) {
+      final RelDataType type = DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName);
+      assertEquals(
+          StringComparators.LEXICOGRAPHIC,
+          Calcites.getStringComparatorForRelDataType(type),
+          type.getFullTypeString()
+      );
+    }
+
+    for (final SqlTypeName typeName : SqlTypeFamily.NUMERIC.getTypeNames()) {
+      final RelDataType type = DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName);
+      assertEquals(
+          StringComparators.NUMERIC,
+          Calcites.getStringComparatorForRelDataType(type),
+          type.getFullTypeString()
+      );
+    }
+
+    assertEquals(
+        StringComparators.NATURAL,
+        Calcites.getStringComparatorForRelDataType(
+            RowSignatures.makeComplexType(DruidTypeSystem.TYPE_FACTORY, ColumnType.UNKNOWN_COMPLEX, false)
+        ),
+        ColumnType.UNKNOWN_COMPLEX.toString()
+    );
+
+    assertEquals(
+        StringComparators.NATURAL,
+        Calcites.getStringComparatorForRelDataType(
+            RowSignatures.makeComplexType(DruidTypeSystem.TYPE_FACTORY, ColumnType.NESTED_DATA, false)
+        ),
+        ColumnType.NESTED_DATA.toString()
+    );
+
+    final RelDataType timestampType = DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP);
+    assertEquals(
+        StringComparators.NUMERIC,
+        Calcites.getStringComparatorForRelDataType(timestampType),
+        timestampType.getFullTypeString()
+    );
+
+    final RelDataType dateType = DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.DATE);
+    assertEquals(
+        StringComparators.NUMERIC,
+        Calcites.getStringComparatorForRelDataType(dateType),
+        dateType.getFullTypeString()
+    );
+
+    final RelDataType bigintArrayType =
+        DruidTypeSystem.TYPE_FACTORY.createArrayType(
+            DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT),
+            -1
+        );
+    assertEquals(
+        StringComparators.NATURAL,
+        Calcites.getStringComparatorForRelDataType(bigintArrayType),
+        bigintArrayType.getFullTypeString()
+    );
+
+    final RelDataType booleanType = DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.BOOLEAN);
+    assertEquals(
+        StringComparators.NUMERIC,
+        Calcites.getStringComparatorForRelDataType(booleanType),
+        booleanType.getFullTypeString()
+    );
+
+    final RelDataType otherType = DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.OTHER);
+    assertEquals(
+        StringComparators.NATURAL,
+        Calcites.getStringComparatorForRelDataType(otherType),
+        otherType.getFullTypeString()
+    );
+
+    final RelDataType nullType = DruidTypeSystem.TYPE_FACTORY.createSqlType(SqlTypeName.NULL);
+    assertEquals(
+        StringComparators.NATURAL,
+        Calcites.getStringComparatorForRelDataType(nullType),
+        nullType.getFullTypeString()
+    );
   }
 
   @Test
   public void testGetStringComparatorForColumnType()
   {
-    Assert.assertEquals(StringComparators.LEXICOGRAPHIC, Calcites.getStringComparatorForValueType(ColumnType.STRING));
-    Assert.assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.LONG));
-    Assert.assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.FLOAT));
-    Assert.assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.DOUBLE));
-    Assert.assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.STRING_ARRAY));
-    Assert.assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.LONG_ARRAY));
-    Assert.assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.DOUBLE_ARRAY));
-    Assert.assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.NESTED_DATA));
-    Assert.assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.UNKNOWN_COMPLEX));
+    assertEquals(StringComparators.LEXICOGRAPHIC, Calcites.getStringComparatorForValueType(ColumnType.STRING));
+    assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.LONG));
+    assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.FLOAT));
+    assertEquals(StringComparators.NUMERIC, Calcites.getStringComparatorForValueType(ColumnType.DOUBLE));
+    assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.STRING_ARRAY));
+    assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.LONG_ARRAY));
+    assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.DOUBLE_ARRAY));
+    assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.NESTED_DATA));
+    assertEquals(StringComparators.NATURAL, Calcites.getStringComparatorForValueType(ColumnType.UNKNOWN_COMPLEX));
   }
 }