Skip to content

Commit

Permalink
Enable resultset validation of Drill tests (#15096)
Browse files Browse the repository at this point in the history
- introduces a test_X method for every testcase (995 testcases)
- added a resultset parser which reads the expected resultset based on the result schema
- loaded a few more datasets
- added a testcase to ensure that all files have a corresponding testcase
- renamed DecoupledIgnore to NegativeTest
- categorized the failing 268 tests
  • Loading branch information
kgyrtkirk authored and LakshSingla committed Oct 13, 2023
1 parent 97a70df commit 6dbd556
Show file tree
Hide file tree
Showing 7 changed files with 7,978 additions and 140 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
Expand Down Expand Up @@ -1056,7 +1057,8 @@ public Map<String, Object> baseQueryContext()

public void assertResultsEquals(String sql, List<Object[]> expectedResults, List<Object[]> results)
{
for (int i = 0; i < results.size(); i++) {
int minSize = Math.min(results.size(), expectedResults.size());
for (int i = 0; i < minSize; i++) {
Assert.assertArrayEquals(
StringUtils.format("result #%d: %s", i + 1, sql),
expectedResults.get(i),
Expand Down Expand Up @@ -1368,7 +1370,7 @@ public void verify(String sql, List<Object[]> results)
assertResultsEquals(sql, expectedResults, results);
}
catch (AssertionError e) {
displayResults(results);
displayResults("Actual", results);
throw e;
}
}
Expand All @@ -1380,10 +1382,10 @@ public void verify(String sql, List<Object[]> results)
* expected results: let the test fail with empty results. The actual results
* are printed to the console. Copy them into the test.
*/
public static void displayResults(List<Object[]> results)
public static void displayResults(String name, List<Object[]> results)
{
PrintStream out = System.out;
out.println("-- Actual results --");
out.printf(Locale.ENGLISH, "-- %s results --", name);
for (int rowIndex = 0; rowIndex < results.size(); rowIndex++) {
printArray(results.get(rowIndex), out);
if (rowIndex < results.size() - 1) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.join.JoinType;
import org.apache.druid.sql.calcite.DecoupledIgnore.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.Modes;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.filtration.Filtration;
import org.apache.druid.sql.calcite.planner.Calcites;
Expand Down Expand Up @@ -2717,7 +2717,7 @@ public void testGroupByWithSelectProjections()
);
}

@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSelectAndOrderByProjections()
{
Expand Down Expand Up @@ -2802,7 +2802,7 @@ public void testTopNWithSelectProjections()
);
}

@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testTopNWithSelectAndOrderByProjections()
{
Expand Down Expand Up @@ -2840,7 +2840,7 @@ public void testTopNWithSelectAndOrderByProjections()
);
}

@DecoupledIgnore
@NotYetSupported
@Test
public void testUnionAllQueries()
{
Expand Down Expand Up @@ -2874,7 +2874,7 @@ public void testUnionAllQueries()
);
}

@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testUnionAllQueriesWithLimit()
{
Expand Down Expand Up @@ -3232,7 +3232,7 @@ public void testNullFloatFilter()
* This test case should be in {@link CalciteUnionQueryTest}. However, there's a bug in the test framework that
* doesn't reset framework once the merge buffers
*/
@DecoupledIgnore
@NotYetSupported
@Test
public void testUnionAllSameTableThreeTimes()
{
Expand Down Expand Up @@ -3276,7 +3276,7 @@ public void testUnionAllSameTableThreeTimes()
);
}

@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testExactCountDistinctUsingSubqueryOnUnionAllTables()
{
Expand Down Expand Up @@ -4682,7 +4682,7 @@ public void testSimpleAggregations()
);
}

@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationDefault()
{
Expand Down Expand Up @@ -4714,7 +4714,7 @@ public void testGroupByWithSortOnPostAggregationDefault()
);
}

@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationNoTopNConfig()
{
Expand Down Expand Up @@ -4758,7 +4758,7 @@ public void testGroupByWithSortOnPostAggregationNoTopNConfig()
);
}

@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationNoTopNContext()
{
Expand Down Expand Up @@ -5346,7 +5346,7 @@ public void testCountStarWithNotOfDegenerateFilter()
);
}

@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testUnplannableQueries()
{
Expand Down Expand Up @@ -5418,7 +5418,7 @@ public void testCountStarWithBoundFilterSimplifyOr()
);
}

@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testUnplannableExactCountDistinctOnSketch()
{
Expand Down Expand Up @@ -6413,7 +6413,7 @@ public void testApproxCountDistinctBuiltin()
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testExactCountDistinctWithGroupingAndOtherAggregators()
{
Expand Down Expand Up @@ -6468,7 +6468,7 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators()
);
}

@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testMultipleExactCountDistinctWithGroupingAndOtherAggregatorsUsingJoin()
{
Expand Down Expand Up @@ -7111,7 +7111,7 @@ public void testQueryWithMoreThanMaxNumericInFilter()
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testExactCountDistinctUsingSubqueryWithWherePushDown()
{
Expand Down Expand Up @@ -7842,7 +7842,7 @@ public void testGroupAndFilterOnTimeFloorWithTimeZone()
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testFilterOnCurrentTimestampWithIntervalArithmetic()
{
Expand Down Expand Up @@ -7890,7 +7890,7 @@ public void testFilterOnCurrentTimestampLosAngeles()
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testFilterOnCurrentTimestampOnView()
{
Expand Down Expand Up @@ -10141,7 +10141,7 @@ public void testGroupByTimeAndOtherDimension()
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim()
{
Expand Down Expand Up @@ -11590,7 +11590,7 @@ public void testTextcat()
);
}

@DecoupledIgnore
@NotYetSupported
@Test
public void testRequireTimeConditionPositive()
{
Expand Down Expand Up @@ -11794,7 +11794,7 @@ public void testRequireTimeConditionSubQueryNegative()
);
}

@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testRequireTimeConditionSemiJoinNegative()
{
Expand Down Expand Up @@ -13773,7 +13773,7 @@ public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleCons
);
}

@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testPlanWithInFilterLessThanInSubQueryThreshold()
{
Expand Down Expand Up @@ -14098,7 +14098,7 @@ public void testComplexDecodeAgg()
)
);
}
@DecoupledIgnore
@NotYetSupported
@Test
public void testOrderByAlongWithInternalScanQuery()
{
Expand Down Expand Up @@ -14141,7 +14141,7 @@ public void testOrderByAlongWithInternalScanQuery()
);
}

@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testOrderByAlongWithInternalScanQueryNoDistinct()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@
package org.apache.druid.sql.calcite;

import com.google.common.collect.ImmutableList;
import org.apache.druid.sql.calcite.DecoupledIgnore.DecoupledIgnoreProcessor;
import org.apache.druid.sql.calcite.DecoupledIgnore.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor;
import org.junit.Rule;
import org.junit.Test;

public class CalciteSysQueryTest extends BaseCalciteQueryTest
{
@Rule(order = 0)
public DecoupledIgnoreProcessor decoupledIgnoreProcessor = new DecoupledIgnoreProcessor();
public NotYetSupportedProcessor NegativeTestProcessor = new NotYetSupportedProcessor();

@Test
public void testTasksSum()
Expand All @@ -46,7 +46,7 @@ public void testTasksSum()
.run();
}

@DecoupledIgnore(mode = Modes.EXPRESSION_NOT_GROUPED)
@NotYetSupported(Modes.EXPRESSION_NOT_GROUPED)
@Test
public void testTasksSumOver()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.google.common.collect.ImmutableMap;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.sql.calcite.DecoupledIgnore.DecoupledIgnoreProcessor;
import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.util.SqlTestFramework;
import org.junit.Rule;
Expand All @@ -31,7 +31,7 @@ public class DecoupledPlanningCalciteQueryTest extends CalciteQueryTest
{

@Rule(order = 0)
public DecoupledIgnoreProcessor decoupledIgnoreProcessor = new DecoupledIgnoreProcessor();
public NotYetSupportedProcessor decoupledIgnoreProcessor = new NotYetSupportedProcessor();

private static final ImmutableMap<String, Object> CONTEXT_OVERRIDES = ImmutableMap.of(
PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE, PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED,
Expand Down
55 changes: 55 additions & 0 deletions sql/src/test/java/org/apache/druid/sql/calcite/DisableUnless.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.sql.calcite;

import com.google.common.base.Supplier;
import org.apache.druid.common.config.NullHandling;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;

import static org.junit.Assume.assumeTrue;

/**
* Collection of conditional disabler rules.
*/
class DisableUnless
{
public static final TestRule SQL_COMPATIBLE = new DisableUnlessRule("NullHandling::sqlCompatible", NullHandling::sqlCompatible);

public static class DisableUnlessRule implements TestRule
{
private Supplier<Boolean> predicate;
private String message;

public DisableUnlessRule(String message, Supplier<Boolean> predicate)
{
this.message = message;
this.predicate = predicate;
}

@Override
public Statement apply(Statement base, Description description)
{
assumeTrue("Testcase disabled; because condition not met: " + message, predicate.get());
return base;
}
}
}
Loading

0 comments on commit 6dbd556

Please sign in to comment.