Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Intervals are updated properly for Unnest queries #15020

Merged
merged 13 commits into from
Oct 3, 2023
Merged

Intervals are updated properly for Unnest queries #15020

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
e0e3cca
Intervals are updated properly for Unnest queries
somu-imply Sep 20, 2023
c9454c6
More nested test cases
somu-imply Sep 21, 2023
a03e92a
Minor refactoring to use optimize/optimizeFilterOnly
somu-imply Sep 21, 2023
22e162d
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Sep 21, 2023
5534076
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Sep 22, 2023
3cc4d84
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Sep 25, 2023
f4bdd07
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Sep 25, 2023
14e512a
Adding comment, extra test case and updating an error message
somu-imply Sep 25, 2023
3157667
Defensive check added for filteredDS
somu-imply Sep 26, 2023
73e8ecb
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Sep 26, 2023
cfdc854
Handling for nested cases along with examples
somu-imply Sep 28, 2023
3fef69c
tmp
somu-imply Sep 29, 2023
0f931d0
Merge remote-tracking branch 'upstream/master' into unnest_filterDS
somu-imply Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,12 @@
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.DataSource;
import org.apache.druid.query.FilteredDataSource;
import org.apache.druid.query.JoinDataSource;
import org.apache.druid.query.Query;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.TableDataSource;
import org.apache.druid.query.UnnestDataSource;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.LongMaxAggregatorFactory;
import org.apache.druid.query.aggregation.LongMinAggregatorFactory;
Expand Down Expand Up @@ -788,6 +790,28 @@ static Pair<DataSource, Filtration> getFiltration(
{
if (!canUseIntervalFiltering(dataSource)) {
return Pair.of(dataSource, toFiltration(filter, virtualColumnRegistry.getFullRowSignature(), false));
} else if (dataSource instanceof UnnestDataSource) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: It won't make a difference with the logic as is, but let's put this branch at the top since it is the most specific of the branches.

// UnnestDataSource can have another unnest data source
// join datasource, filtered data source, etc as base
Pair<DataSource, Filtration> pair = getFiltration(
((UnnestDataSource) dataSource).getBase(),
somu-imply marked this conversation as resolved.
Show resolved Hide resolved
filter,
virtualColumnRegistry,
joinableFactoryWrapper
);
return Pair.of(dataSource, pair.rhs);
} else if (dataSource instanceof FilteredDataSource) {
final FilteredDataSource filteredDataSource = (FilteredDataSource) dataSource;
final boolean useIntervalFiltering = canUseIntervalFiltering(filteredDataSource);
final Filtration baseFiltration = toFiltration(
filteredDataSource.getFilter(),
virtualColumnRegistry.getFullRowSignature(),
useIntervalFiltering
);
// Adds the intervals from the filter of filtered data source to query filtration
final Filtration queryFiltration = Filtration.create(filter, baseFiltration.getIntervals())
.optimize(virtualColumnRegistry.getFullRowSignature());
somu-imply marked this conversation as resolved.
Show resolved Hide resolved
return Pair.of(filteredDataSource, queryFiltration);
} else if (dataSource instanceof JoinDataSource && ((JoinDataSource) dataSource).getLeftFilter() != null) {
final JoinDataSource joinDataSource = (JoinDataSource) dataSource;

Expand Down
193 changes: 193 additions & 0 deletions sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.HumanReadableBytes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.math.expr.ExprMacroTable;
Expand Down Expand Up @@ -4869,4 +4870,196 @@ public void testUnnestVirtualWithColumnsAndNullIf()
)
);
}

@Test
public void testUnnestWithTimeFilterOnly()
{
testQuery(
"select c from foo, unnest(MV_TO_ARRAY(dim3)) as u(c)"
+ " where __time >= TIMESTAMP '2000-01-02 00:00:00' and __time <= TIMESTAMP '2000-01-03 00:10:00'",
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE1),
range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false)
),
expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
null
))
.intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z")))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"d"}
)
);
}

@Test
public void testUnnestWithTimeFilterAndAnotherFilter()
{
testQuery(
"select c from foo, unnest(MV_TO_ARRAY(dim3)) as u(c) "
+ " where m1=2 and __time >= TIMESTAMP '2000-01-02 00:00:00' and __time <= TIMESTAMP '2000-01-03 00:10:00'",
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE1),
and(
useDefault ? equality("m1", 2, ColumnType.FLOAT) :
equality("m1", 2.0, ColumnType.FLOAT),
range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false)
)
),
expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
null
))
.intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z")))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"c"}
)
);
}

@Test
public void testUnnestWithTimeFilterOrAnotherFilter()
{
testQuery(
"select c from foo, unnest(MV_TO_ARRAY(dim3)) as u(c) "
+ " where m1=2 or __time >= TIMESTAMP '2000-01-02 00:00:00' and __time <= TIMESTAMP '2000-01-03 00:10:00'",
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE1),
or(
useDefault ? equality("m1", 2, ColumnType.FLOAT) :
equality("m1", 2.0, ColumnType.FLOAT),
range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false)
)
),
expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
null
))
.intervals(querySegmentSpec(Filtration.eternity()))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"d"}
)
);
}

@Test
public void testUnnestWithTimeFilterOnlyNested()
{
testQuery(
"select c from foo CROSS JOIN UNNEST(ARRAY[m1,m2]) as un(d) CROSS JOIN unnest(MV_TO_ARRAY(dim3)) as u(c)"
+ " where __time >= TIMESTAMP '2000-01-02 00:00:00' and __time <= TIMESTAMP '2000-01-03 00:10:00'",
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(UnnestDataSource.create(
UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE1),
range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false)
),
expressionVirtualColumn("j0.unnest", "array(\"m1\",\"m2\")", ColumnType.FLOAT_ARRAY),
null
),
expressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING),
null
))
.intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z")))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("_j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"d"},
new Object[]{"d"}
)
);
}

@Test
public void testUnnestWithTimeFilterOnlyNestedAndNestedAgain()
{
testQuery(
"select c from foo CROSS JOIN UNNEST(ARRAY[m1,m2]) as un(d) CROSS JOIN UNNEST(ARRAY[dim1,dim2]) as ud(a) "
+ " CROSS JOIN unnest(MV_TO_ARRAY(dim3)) as u(c)"
+ " where __time >= TIMESTAMP '2000-01-02 00:00:00' and __time <= TIMESTAMP '2000-01-03 00:10:00'",
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(UnnestDataSource.create(
UnnestDataSource.create(
UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE1),
range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false)
),
expressionVirtualColumn("j0.unnest", "array(\"m1\",\"m2\")", ColumnType.FLOAT_ARRAY),
null
),
expressionVirtualColumn("_j0.unnest", "array(\"dim1\",\"dim2\")", ColumnType.STRING_ARRAY),
null
),
expressionVirtualColumn("__j0.unnest", "\"dim3\"", ColumnType.STRING),
null
))
.intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z")))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("__j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"b"},
new Object[]{"c"},
new Object[]{"d"},
new Object[]{"d"},
new Object[]{"d"},
new Object[]{"d"}
)
);
}
}