Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable AggregateProjectMergeRule #15003

Closed
wants to merge 14 commits into from
Closed

Enable AggregateProjectMergeRule #15003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
af9e3ed
Enable already passing tests in DecoupledPlanningCalciteQueryTest
kgyrtkirk Sep 15, 2023
b22bd26
empty
kgyrtkirk Sep 15, 2023
1644321
add transposerule; fix prefix
kgyrtkirk Sep 18, 2023
38bfec3
accept a few more
kgyrtkirk Sep 18, 2023
2b6385d
add projectmerge;aggrproject
kgyrtkirk Sep 18, 2023
eaa5a20
fix order
kgyrtkirk Sep 18, 2023
4ae44c2
Merge remote-tracking branch 'apache/master' into decouple-test-fixes-2
kgyrtkirk Jan 17, 2024
8c1b235
not needed
kgyrtkirk Jan 17, 2024
206e556
undo postAgg name change for now; fix 1 test
kgyrtkirk Jan 17, 2024
e8498f8
some changes
kgyrtkirk Jan 18, 2024
8ceb566
kinda back to orig diffs
kgyrtkirk Jan 18, 2024
2e99c01
some changes
kgyrtkirk Jan 18, 2024
0fd791c
x
kgyrtkirk Jan 18, 2024
454afd3
more trials
kgyrtkirk Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion processing/src/main/java/org/apache/druid/query/QueryContexts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public class QueryContexts
public static final boolean DEFAULT_ENABLE_SQL_JOIN_LEFT_SCAN_DIRECT = false;
public static final boolean DEFAULT_USE_FILTER_CNF = false;
public static final boolean DEFAULT_SECONDARY_PARTITION_PRUNING = true;
public static final boolean DEFAULT_ENABLE_DEBUG = false;
public static final boolean DEFAULT_ENABLE_DEBUG = true;
public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
public static final boolean DEFAULT_ENABLE_TIME_BOUNDARY_PLANNING = false;

Expand Down
8 changes: 8 additions & 0 deletions sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,13 @@ public class CalciteRulesManager
CoreRules.FILTER_VALUES_MERGE,
CoreRules.PROJECT_FILTER_VALUES_MERGE,
CoreRules.PROJECT_VALUES_MERGE,
CoreRules.PROJECT_MERGE,
CoreRules.FILTER_MERGE,
// CoreRules.AGGREGATE_PROJECT_MERGE,
// CoreRules.PROJECT_FILTER_TRANSPOSE,
CoreRules.FILTER_PROJECT_TRANSPOSE,
CoreRules.PROJECT_MERGE,
CoreRules.SORT_PROJECT_TRANSPOSE,
CoreRules.AGGREGATE_VALUES
);

Expand Down Expand Up @@ -426,6 +433,7 @@ public List<RelOptRule> baseRuleSet(final PlannerContext plannerContext)
final ImmutableList.Builder<RelOptRule> rules = ImmutableList.builder();

// Calcite rules.

rules.addAll(BASE_RULES);
rules.addAll(ABSTRACT_RULES);
rules.addAll(ABSTRACT_RELATIONAL_RULES);
Expand Down
148 changes: 98 additions & 50 deletions sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6815,7 +6815,6 @@ public void testApproxCountDistinctBuiltin()
);
}

@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testExactCountDistinctWithGroupingAndOtherAggregators()
{
Expand All @@ -6834,8 +6833,8 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators()
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setDimensions(dimensions(
new DefaultDimensionSpec("dim2", "d0"),
new DefaultDimensionSpec("dim1", "d1")
new DefaultDimensionSpec("dim1", "d0"),
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

grouping columns should come first

new DefaultDimensionSpec("dim2", "d1")
))
.setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt")))
.setContext(QUERY_CONTEXT_DEFAULT)
Expand All @@ -6844,12 +6843,12 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators()
)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setDimensions(dimensions(new DefaultDimensionSpec("d0", "_d0")))
.setDimensions(dimensions(new DefaultDimensionSpec("d1", "_d0")))
.setAggregatorSpecs(aggregators(
new LongSumAggregatorFactory("_a0", "a0"),
new FilteredAggregatorFactory(
new CountAggregatorFactory("_a1"),
notNull("d1")
notNull("d0")
)
))
.setContext(QUERY_CONTEXT_DEFAULT)
Expand Down Expand Up @@ -8080,8 +8079,8 @@ public void testGroupBySortPushDown()
.setGranularity(Granularities.ALL)
.setDimensions(
dimensions(
new DefaultDimensionSpec("dim2", "d0"),
new DefaultDimensionSpec("dim1", "d1")
new DefaultDimensionSpec("dim1", "d0"),
new DefaultDimensionSpec("dim2", "d1")
)
)
.setAggregatorSpecs(
Expand All @@ -8092,7 +8091,7 @@ public void testGroupBySortPushDown()
.setLimitSpec(
new DefaultLimitSpec(
ImmutableList.of(
new OrderByColumnSpec("d1", OrderByColumnSpec.Direction.ASCENDING)
new OrderByColumnSpec("d0", OrderByColumnSpec.Direction.ASCENDING)
),
4
)
Expand Down Expand Up @@ -14085,61 +14084,78 @@ public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithSingleConsta
),
ImmutableList.of()
);
}
@Test
public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithSingleConstantDimension2()
{
skipVectorize();


// dim1 is not getting reduced to 'wat' in this case in Calcite (ProjectMergeRule is not getting applied),
// therefore the query is not optimized to a timeseries query
testQuery(
"SELECT 'A' from foo WHERE dim1 = 'wat' GROUP BY dim1",
ImmutableList.of(
GroupByQuery.builder()
.setDataSource(
"foo"
)
.setInterval(querySegmentSpec(Intervals.ETERNITY))
.setGranularity(Granularities.ALL)
.addDimension(new DefaultDimensionSpec("dim1", "d0", ColumnType.STRING))
.setDimFilter(equality("dim1", "wat", ColumnType.STRING))
.setPostAggregatorSpecs(
ImmutableList.of(
expressionPostAgg("p0", "'A'", ColumnType.STRING)
)
)

.build()
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
equality("dim1", "wat", ColumnType.STRING)
)
.granularity(Granularities.ALL)
.postAggregators(
expressionPostAgg("p0", "'A'", ColumnType.STRING)
)
.context(QUERY_CONTEXT_DO_SKIP_EMPTY_BUCKETS)
.build()
),
ImmutableList.of()
);
}

public void testQuery1(
final String sql,
final List<Query<?>> expectedQueries,
final List<Object[]> expectedResults
)
{
testBuilder()
.sql(sql)
.expectedQueries(expectedQueries)
.expectedResults(expectedResults)
.run();
}

@Test
public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleConstantDimensions()
{
skipVectorize();
testQuery(
"SELECT 'A', dim1 from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY dim1",
ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
and(
NullHandling.replaceWithDefault()
? selector("m1", "50")
: equality("m1", 50.0, ColumnType.FLOAT),
equality("dim1", "wat", ColumnType.STRING)
testBuilder()
.sql("SELECT 'A', dim1 from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY dim1")
.queryContext(
ImmutableMap.of(
QueryContexts.ENABLE_DEBUG, true
)
)
.expectedQueries(ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
and(
NullHandling.replaceWithDefault()
? selector("m1", "50")
: equality("m1", 50.0, ColumnType.FLOAT),
equality("dim1", "wat", ColumnType.STRING)
)
)
)
.granularity(Granularities.ALL)
.postAggregators(
expressionPostAgg("p0", "'A'", ColumnType.STRING),
expressionPostAgg("p1", "'wat'", ColumnType.STRING)
)
.context(QUERY_CONTEXT_DO_SKIP_EMPTY_BUCKETS)
.build()
),
ImmutableList.of()
);
.granularity(Granularities.ALL)
.postAggregators(
expressionPostAgg("p0", "'A'", ColumnType.STRING),
expressionPostAgg("p1", "'wat'", ColumnType.STRING)
)
.context(QUERY_CONTEXT_DO_SKIP_EMPTY_BUCKETS)
.build()
))
.expectedResults(ImmutableList.of())
.run();

// Sanity test, that even when dimensions are reduced, but should produce a valid result (i.e. when filters are
// correct, then they should
Expand Down Expand Up @@ -14167,7 +14183,6 @@ public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleCons
);
}

@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testPlanWithInFilterLessThanInSubQueryThreshold()
{
Expand Down Expand Up @@ -15039,6 +15054,39 @@ public void testWindowingWithScanAndSort()
.run();
}


@NotYetSupported(Modes.CANNOT_TRANSLATE)
@Test
public void testWindowingWithScanAndSortX()
{
skipVectorize();
cannotVectorize();
msqIncompatible();
String sql = "\n"
+ "SELECT dim1,dim2,count(1),count(1) OVER (partition by dim2) FROM "
+ " (select dim1,dim2 from foo GROUP BY dim1,dim2 order by dim2) t group by dim1,dim2";
// + "SELECT dim1,dim2,count(1) FROM foo GROUP BY dim1,dim2 order by dim2";
ImmutableList<Object[]> expectedResults = ImmutableList.of(
new Object[]{"", "a", 1L},
new Object[]{"1", "a", 1L},
new Object[]{"10.1", null, 1L},
new Object[]{"2", "", 1L},
new Object[]{"abc", null, 1L},
new Object[]{"def", "abc", 1L}
);

testBuilder()
.queryContext(
ImmutableMap.of(
PlannerContext.CTX_ENABLE_WINDOW_FNS, true,
QueryContexts.ENABLE_DEBUG, true
)
)
.sql(sql)
.expectedResults(expectedResults)
.run();
}

@NotYetSupported(Modes.CANNOT_TRANSLATE)
@Test
public void testWindowingWithOrderBy()
Expand Down
Loading