diff --git a/src/db.rs b/src/db.rs index bf0f4eec..30361fd6 100644 --- a/src/db.rs +++ b/src/db.rs @@ -201,22 +201,21 @@ impl Database { ], ) .batch( - "Combine Operators".to_string(), + "Limit Pushdown".to_string(), HepBatchStrategy::fix_point_topdown(10), vec![ - NormalizationRuleImpl::CollapseProject, - NormalizationRuleImpl::CollapseGroupByAgg, - NormalizationRuleImpl::CombineFilter, + NormalizationRuleImpl::LimitProjectTranspose, + NormalizationRuleImpl::PushLimitThroughJoin, + NormalizationRuleImpl::PushLimitIntoTableScan, ], ) .batch( - "Limit Pushdown".to_string(), + "Combine Operators".to_string(), HepBatchStrategy::fix_point_topdown(10), vec![ - NormalizationRuleImpl::LimitProjectTranspose, - NormalizationRuleImpl::PushLimitThroughJoin, - NormalizationRuleImpl::PushLimitIntoTableScan, - NormalizationRuleImpl::EliminateLimits, + NormalizationRuleImpl::CollapseProject, + NormalizationRuleImpl::CollapseGroupByAgg, + NormalizationRuleImpl::CombineFilter, ], ) .batch( @@ -364,7 +363,9 @@ mod test { fn test_udtf() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); let fnck_sql = DataBaseBuilder::path(temp_dir.path()).build()?; - let (schema, tuples) = fnck_sql.run("select number from table(numbers(10))")?; + let (schema, tuples) = fnck_sql.run( + "SELECT * FROM (select * from table(numbers(10)) a ORDER BY number LIMIT 5) OFFSET 3", + )?; println!("{}", create_table(&schema, &tuples)); Ok(()) diff --git a/src/execution/dql/limit.rs b/src/execution/dql/limit.rs index ad4a94ce..4d4a459c 100644 --- a/src/execution/dql/limit.rs +++ b/src/execution/dql/limit.rs @@ -42,7 +42,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Limit { } let offset_val = offset.unwrap_or(0); - let offset_limit = offset_val + limit.unwrap_or(1) - 1; + let offset_limit = offset_val.saturating_add(limit.unwrap_or(usize::MAX)) - 1; let mut i = 0; let mut coroutine = build_read(input, cache, transaction); diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index d0ee07f6..8c30fd4a 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -11,7 +11,7 @@ use crate::optimizer::rule::normalization::compilation_in_advance::{ EvaluatorBind, ExpressionRemapper, }; use crate::optimizer::rule::normalization::pushdown_limit::{ - EliminateLimits, LimitProjectTranspose, PushLimitIntoScan, PushLimitThroughJoin, + LimitProjectTranspose, PushLimitIntoScan, PushLimitThroughJoin, }; use crate::optimizer::rule::normalization::pushdown_predicates::PushPredicateIntoScan; use crate::optimizer::rule::normalization::pushdown_predicates::PushPredicateThroughJoin; @@ -34,7 +34,6 @@ pub enum NormalizationRuleImpl { CombineFilter, // PushDown limit LimitProjectTranspose, - EliminateLimits, PushLimitThroughJoin, PushLimitIntoTableScan, // PushDown predicates @@ -57,7 +56,6 @@ impl MatchPattern for NormalizationRuleImpl { NormalizationRuleImpl::CollapseGroupByAgg => CollapseGroupByAgg.pattern(), NormalizationRuleImpl::CombineFilter => CombineFilter.pattern(), NormalizationRuleImpl::LimitProjectTranspose => LimitProjectTranspose.pattern(), - NormalizationRuleImpl::EliminateLimits => EliminateLimits.pattern(), NormalizationRuleImpl::PushLimitThroughJoin => PushLimitThroughJoin.pattern(), NormalizationRuleImpl::PushLimitIntoTableScan => PushLimitIntoScan.pattern(), NormalizationRuleImpl::PushPredicateThroughJoin => PushPredicateThroughJoin.pattern(), @@ -80,7 +78,6 @@ impl NormalizationRule for NormalizationRuleImpl { NormalizationRuleImpl::LimitProjectTranspose => { LimitProjectTranspose.apply(node_id, graph) } - NormalizationRuleImpl::EliminateLimits => EliminateLimits.apply(node_id, graph), NormalizationRuleImpl::PushLimitThroughJoin => { PushLimitThroughJoin.apply(node_id, graph) } diff --git a/src/optimizer/rule/normalization/pushdown_limit.rs b/src/optimizer/rule/normalization/pushdown_limit.rs index 33573bd3..d3f421ab 100644 --- a/src/optimizer/rule/normalization/pushdown_limit.rs +++ b/src/optimizer/rule/normalization/pushdown_limit.rs @@ -4,11 +4,10 @@ use crate::optimizer::core::pattern::PatternChildrenPredicate; use crate::optimizer::core::rule::{MatchPattern, NormalizationRule}; use crate::optimizer::heuristic::graph::{HepGraph, HepNodeId}; use crate::planner::operator::join::JoinType; -use crate::planner::operator::limit::LimitOperator; use crate::planner::operator::Operator; use itertools::Itertools; use lazy_static::lazy_static; -use std::cmp; + lazy_static! { static ref LIMIT_PROJECT_TRANSPOSE_RULE: Pattern = { Pattern { @@ -66,51 +65,6 @@ impl NormalizationRule for LimitProjectTranspose { } } -/// Combines two adjacent Limit operators into one, merging the expressions into one single -/// expression. -pub struct EliminateLimits; - -impl MatchPattern for EliminateLimits { - fn pattern(&self) -> &Pattern { - &ELIMINATE_LIMITS_RULE - } -} - -impl NormalizationRule for EliminateLimits { - fn apply(&self, node_id: HepNodeId, graph: &mut HepGraph) -> Result<(), DatabaseError> { - if let Operator::Limit(op) = graph.operator(node_id) { - if let Some(child_id) = graph.eldest_child_at(node_id) { - if let Operator::Limit(child_op) = graph.operator(child_id) { - let offset = Self::binary_options(op.offset, child_op.offset, |a, b| a + b); - let limit = Self::binary_options(op.limit, child_op.limit, cmp::min); - - let new_limit_op = LimitOperator { offset, limit }; - - graph.remove_node(child_id, false); - graph.replace_node(node_id, Operator::Limit(new_limit_op)); - } - } - } - - Ok(()) - } -} - -impl EliminateLimits { - fn binary_options usize>( - a: Option, - b: Option, - _fn: F, - ) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(_fn(a, b)), - (Some(a), None) => Some(a), - (None, Some(b)) => Some(b), - (None, None) => None, - } - } -} - /// Add extra limits below JOIN: /// 1. For LEFT OUTER and RIGHT OUTER JOIN, we push limits to the left and right sides, /// respectively. @@ -190,7 +144,6 @@ mod tests { use crate::optimizer::heuristic::batch::HepBatchStrategy; use crate::optimizer::heuristic::optimizer::HepOptimizer; use crate::optimizer::rule::normalization::NormalizationRuleImpl; - use crate::planner::operator::limit::LimitOperator; use crate::planner::operator::Operator; use crate::storage::rocksdb::RocksTransaction; @@ -219,39 +172,6 @@ mod tests { Ok(()) } - #[test] - fn test_eliminate_limits() -> Result<(), DatabaseError> { - let plan = select_sql_run("select c1, c2 from t1 limit 1 offset 1")?; - - let mut optimizer = HepOptimizer::new(plan.clone()).batch( - "test_eliminate_limits".to_string(), - HepBatchStrategy::once_topdown(), - vec![NormalizationRuleImpl::EliminateLimits], - ); - - let new_limit_op = LimitOperator { - offset: Some(2), - limit: Some(1), - }; - - optimizer.graph.add_root(Operator::Limit(new_limit_op)); - - let best_plan = optimizer.find_best::(None)?; - - if let Operator::Limit(op) = &best_plan.operator { - assert_eq!(op.limit, Some(1)); - assert_eq!(op.offset, Some(3)); - } else { - unreachable!("Should be a project operator") - } - - if let Operator::Limit(_) = &best_plan.childrens[0].operator { - unreachable!("Should not be a limit operator") - } - - Ok(()) - } - #[test] fn test_push_limit_through_join() -> Result<(), DatabaseError> { let plan = select_sql_run("select * from t1 left join t2 on c1 = c3 limit 1")?; diff --git a/tests/slt/crdb/limit.slt b/tests/slt/crdb/limit.slt new file mode 100644 index 00000000..bf2f2df9 --- /dev/null +++ b/tests/slt/crdb/limit.slt @@ -0,0 +1,197 @@ +query I +SELECT number FROM table(numbers(100)) ORDER BY number LIMIT 5 +---- +0 +1 +2 +3 +4 + +query I +SELECT number FROM table(numbers(100)) ORDER BY number limit 5 +---- +0 +1 +2 +3 +4 + +query I +SELECT number FROM table(numbers(100)) ORDER BY number limit 1 +---- +0 + +query I +SELECT number FROM table(numbers(100)) ORDER BY number offset 1 limit 3 +---- +1 +2 +3 + +query I +SELECT number FROM table(numbers(100)) ORDER BY number limit 1 OFFSET 3 +---- +3 + +# is order limit and offset matters? +# query I +# SELECT number FROM table(numbers(100)) ORDER BY number OFFSET 3 limit 1 + +# ---- +# 3 + +query I +SELECT number FROM table(numbers(100)) ORDER BY number limit 2 +---- +0 +1 + +statement ok +drop table if exists t + +statement ok +CREATE TABLE t (k INT PRIMARY KEY, v INT, w INT) + +statement ok +INSERT INTO t VALUES (1, 1, 1), (2, -4, 8), (3, 9, 27), (4, -16, 94), (5, 25, 125), (6, -36, 216) + +query III +SELECT * FROM t WHERE v > -20 AND w > 30 ORDER BY v LIMIT 2 +---- +4 -16 94 +5 25 125 + +query II +SELECT k, v FROM t ORDER BY k LIMIT 5 +---- +1 1 +2 -4 +3 9 +4 -16 +5 25 + +query II +SELECT k, v FROM t ORDER BY k OFFSET 5 +---- +6 -36 + +query II rowsort +SELECT k, v FROM t ORDER BY v LIMIT 5 OFFSET 1 +---- +1 1 +2 -4 +3 9 +4 -16 +5 25 + +query II rowsort +SELECT k, v FROM t ORDER BY v DESC LIMIT 5 OFFSET 1 +---- +1 1 +2 -4 +3 9 +4 -16 +6 -36 + +query I +SELECT k, v, sum(w) FROM t GROUP BY k, v ORDER BY v DESC LIMIT 10 +---- +5 25 125 +3 9 27 +1 1 1 +2 -4 8 +4 -16 94 +6 -36 216 + +query I rowsort +SELECT k FROM (SELECT k, v FROM t ORDER BY v LIMIT 4) +---- +1 +2 +4 +6 + +query I rowsort +SELECT k FROM (SELECT k, v, w FROM t ORDER BY v LIMIT 4) +---- +1 +2 +4 +6 + +query II +SELECT k, v FROM t ORDER BY k LIMIT 6 +---- +1 1 +2 -4 +3 9 +4 -16 +5 25 +6 -36 + +query II +SELECT k, v FROM t ORDER BY k LIMIT 2 +---- +1 1 +2 -4 + +query II rowsort +SELECT k, v FROM t ORDER BY k OFFSET 3 +---- +4 -16 +5 25 +6 -36 + +query II +SELECT k, v FROM t ORDER BY k LIMIT 3 OFFSET 3 +---- +4 -16 +5 25 +6 -36 + +query I +SELECT * FROM (select * from table(numbers(10)) a ORDER BY number LIMIT 5) OFFSET 3 +---- +3 +4 + +statement ok +SELECT * FROM (select * from table(numbers(10)) a LIMIT 5) OFFSET 6 + +statement ok +drop table if exists t_47283 + +statement ok +CREATE TABLE t_47283(k INT PRIMARY KEY, a INT) + +statement ok +INSERT INTO t_47283 VALUES (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6) + +# This should return no results if it does, we incorrectly removed the hard +statement ok +SELECT * FROM (SELECT * FROM t_47283 ORDER BY k LIMIT 4) WHERE a > 5 LIMIT 1 + +# order by expr | limit expr is not support +# SELECT a FROM probe ORDER BY a LIMIT (SELECT v FROM vals WHERE k = 'maxint64') OFFSET (SELECT v FROM vals WHERE k = 'large') + +statement ok +drop table if exists t65171 + +statement ok +CREATE TABLE t65171 (id INT PRIMARY KEY, x INT, y INT) + +statement ok +INSERT INTO t65171 VALUES (0, 1, 2), (1, 1, 2), (2, 2, 3) + +query II +SELECT * FROM t65171 WHERE x = 1 OR x = 2 ORDER BY y LIMIT 2 +---- +0 1 2 +1 1 2 + +query III rowsort +SELECT * FROM t ORDER BY v, w LIMIT 3 +---- +2 -4 8 +4 -16 94 +6 -36 216 \ No newline at end of file