From 2cd9e3979e29ca0510a6a269452f873588ae1c0a Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 13 Mar 2024 22:45:40 +0000 Subject: [PATCH 01/16] feat(tql): add initial support for start,stop,step as sql functions --- src/sql/Cargo.toml | 5 + src/sql/src/parsers/tql_parser.rs | 156 +++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 4 deletions(-) diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml index dcce4b698763..f1b2356a6c79 100644 --- a/src/sql/Cargo.toml +++ b/src/sql/Cargo.toml @@ -16,7 +16,12 @@ common-error.workspace = true common-macro.workspace = true common-query.workspace = true common-time.workspace = true +chrono.workspace = true +datafusion-common.workspace = true +datafusion-expr.workspace = true +datafusion-physical-expr.workspace = true datafusion-sql.workspace = true +datafusion.workspace = true datatypes.workspace = true hex = "0.4" itertools.workspace = true diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 843c54b47b2b..58c212f4aaa8 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -12,6 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + +use chrono::Utc; +use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::{DFSchema, Result as DFResult, ScalarValue, TableReference}; +use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; +use datafusion_physical_expr::execution_props::ExecutionProps; +use datafusion_sql::planner::{ContextProvider, SqlToRel}; use snafu::ResultExt; use sqlparser::keywords::Keyword; use sqlparser::parser::ParserError; @@ -28,8 +37,11 @@ const EVALUATE: &str = "EVALUATE"; const EXPLAIN: &str = "EXPLAIN"; const VERBOSE: &str = "VERBOSE"; +use datatypes::arrow::datatypes::DataType; use sqlparser::parser::Parser; +use crate::dialect::GreptimeDbDialect; + /// TQL extension parser, including: /// - `TQL EVAL ` /// - `TQL EXPLAIN [VERBOSE] ` @@ -69,9 +81,9 @@ impl<'a> ParserContext<'a> { fn parse_tql_eval(&mut self) -> std::result::Result { let parser = &mut self.parser; parser.expect_token(&Token::LParen)?; - let start = Self::parse_string_or_number(parser, Token::Comma)?; - let end = Self::parse_string_or_number(parser, Token::Comma)?; - let step = Self::parse_string_or_number(parser, Token::RParen)?; + let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; let query = Self::parse_tql_query(parser, self.sql, ")")?; Ok(Statement::Tql(Tql::Eval(TqlEval { @@ -100,6 +112,62 @@ impl<'a> ParserContext<'a> { Ok(value) } + fn parse_string_or_number_or_word( + parser: &mut Parser, + token: Token, + ) -> std::result::Result { + let value = match parser.next_token().token { + Token::Number(n, _) => n, + Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, + Token::Word(w) => Self::parse_tql_word(Token::Word(w), parser).unwrap(), + unexpected => { + return Err(ParserError::ParserError(format!( + "Expect number or string TODO adjust message, but is {unexpected:?}" + ))); + } + }; + parser.expect_token(&token)?; + Ok(value) + } + + fn parse_tql_word(w: Token, parser: &mut Parser) -> std::result::Result { + let tokens = Self::collect_tokens(w, parser); + let empty_df_schema = DFSchema::empty(); + + let expr = Parser::new(&GreptimeDbDialect {}) + .with_tokens(tokens) + .parse_expr() + .map_err(|err| { + ParserError::ParserError(format!("Expect number or string, but is {err:?}")) + })?; + + let sql_to_rel = SqlToRel::new(&DummyContextProvider {}); + let logical_expr = sql_to_rel + .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) + .unwrap(); + + let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); + let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); + let simplified_expr = ExprSimplifier::new(info).simplify(logical_expr).unwrap(); + + let timestamp_nanos = match simplified_expr { + Expr::Literal(ScalarValue::TimestampNanosecond(Some(v), _)) => v, + _ => panic!(), + }; + + Ok((timestamp_nanos / 1_000_000_000).to_string()) + } + + fn collect_tokens(w: Token, parser: &mut Parser) -> Vec { + let mut tokens = vec![]; + tokens.push(w); + while parser.peek_token() != Token::Comma { + let token = parser.next_token(); + tokens.push(token.token); + } + tokens + } + fn parse_tql_query( parser: &mut Parser, sql: &str, @@ -113,7 +181,14 @@ impl<'a> ParserContext<'a> { return Err(ParserError::ParserError("empty TQL query".to_string())); } - let query = &sql[index..]; + let query = if delimiter == ")" { + match Self::find_last_balanced_bracket(sql) { + Some(to) => &sql[(to + 1)..], + None => &sql[index..], + } + } else { + &sql[index..] + }; while parser.next_token() != Token::EOF { // consume all tokens @@ -127,6 +202,23 @@ impl<'a> ParserContext<'a> { } } + fn find_last_balanced_bracket(sql: &str) -> Option { + let mut balance = 0; + for (index, c) in sql.char_indices() { + match c { + '(' => balance += 1, + ')' => { + balance -= 1; + if balance == 0 { + return Some(index); + } + } + _ => {} + } + } + None + } + fn parse_tql_explain(&mut self) -> Result { let parser = &mut self.parser; let is_verbose = if parser.peek_token().token.to_string() == VERBOSE { @@ -184,6 +276,35 @@ impl<'a> ParserContext<'a> { } } +#[derive(Default)] +struct DummyContextProvider {} + +impl ContextProvider for DummyContextProvider { + fn get_table_provider(&self, _name: TableReference) -> DFResult> { + unimplemented!() + } + + fn get_function_meta(&self, _name: &str) -> Option> { + None + } + + fn get_aggregate_meta(&self, _name: &str) -> Option> { + unimplemented!() + } + + fn get_window_meta(&self, _name: &str) -> Option> { + unimplemented!() + } + + fn get_variable_type(&self, _variable_names: &[String]) -> Option { + unimplemented!() + } + + fn options(&self) -> &ConfigOptions { + unimplemented!() + } +} + #[cfg(test)] mod tests { use common_error::ext::ErrorExt; @@ -191,6 +312,33 @@ mod tests { use super::*; use crate::dialect::GreptimeDbDialect; use crate::parser::ParseOptions; + + #[test] + fn test_parse_tql_eval_with_functions() { + let sql = "TQL EVAL (now() - '10 minutes'::interval, now(), '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + let mut result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + assert_eq!(1, result.len()); + + let assert_time = Utc::now() + .timestamp_nanos_opt() + .map(|x| x / 1_000_000_000) + .unwrap_or(0); + + let statement = result.remove(0); + + match statement { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, (assert_time - 600).to_string()); + assert_eq!(eval.end, assert_time.to_string()); + assert_eq!(eval.step, "1m"); + assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + } + _ => unreachable!(), + } + } + #[test] fn test_parse_tql_eval() { let sql = "TQL EVAL (1676887657, 1676887659, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; From bc1a549c13f1dd7c196122ce676a10b3caef017c Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 14 Mar 2024 09:31:41 +0000 Subject: [PATCH 02/16] fix(tql): remove unwraps, adjust fmt --- src/sql/Cargo.toml | 2 +- src/sql/src/parsers/tql_parser.rs | 33 ++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml index f1b2356a6c79..b3ae8155b4e4 100644 --- a/src/sql/Cargo.toml +++ b/src/sql/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] api.workspace = true +chrono.workspace = true common-base.workspace = true common-catalog.workspace = true common-decimal.workspace = true @@ -16,7 +17,6 @@ common-error.workspace = true common-macro.workspace = true common-query.workspace = true common-time.workspace = true -chrono.workspace = true datafusion-common.workspace = true datafusion-expr.workspace = true datafusion-physical-expr.workspace = true diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 58c212f4aaa8..a353d8c0db25 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -112,6 +112,7 @@ impl<'a> ParserContext<'a> { Ok(value) } + // TODO this method will substitute `parse_string_or_number` eventually fn parse_string_or_number_or_word( parser: &mut Parser, token: Token, @@ -119,10 +120,10 @@ impl<'a> ParserContext<'a> { let value = match parser.next_token().token { Token::Number(n, _) => n, Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, - Token::Word(w) => Self::parse_tql_word(Token::Word(w), parser).unwrap(), + Token::Word(w) => Self::parse_tql_word(Token::Word(w), parser)?, unexpected => { return Err(ParserError::ParserError(format!( - "Expect number or string TODO adjust message, but is {unexpected:?}" + "Expect number, string or word, but is {unexpected:?}" ))); } }; @@ -133,29 +134,37 @@ impl<'a> ParserContext<'a> { fn parse_tql_word(w: Token, parser: &mut Parser) -> std::result::Result { let tokens = Self::collect_tokens(w, parser); let empty_df_schema = DFSchema::empty(); + let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); let expr = Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) .parse_expr() .map_err(|err| { - ParserError::ParserError(format!("Expect number or string, but is {err:?}")) + ParserError::ParserError(format!("Failed to convert to expression: {err:?}")) })?; let sql_to_rel = SqlToRel::new(&DummyContextProvider {}); let logical_expr = sql_to_rel .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) - .unwrap(); + .map_err(|err| { + ParserError::ParserError(format!("Failed to convert to logical expression {err:?}")) + })?; - let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); - let simplified_expr = ExprSimplifier::new(info).simplify(logical_expr).unwrap(); - - let timestamp_nanos = match simplified_expr { - Expr::Literal(ScalarValue::TimestampNanosecond(Some(v), _)) => v, - _ => panic!(), - }; + let simplified_expr = ExprSimplifier::new(info) + .simplify(logical_expr) + .map_err(|err| { + ParserError::ParserError(format!("Failed to simplify expression {err:?}")) + })?; - Ok((timestamp_nanos / 1_000_000_000).to_string()) + match simplified_expr { + Expr::Literal(ScalarValue::TimestampNanosecond(v, _)) => v, + _ => None, + } + .map(|timestamp_nanos| (timestamp_nanos / 1_000_000_000).to_string()) + .ok_or(ParserError::ParserError(format!( + "Failed to extract a timestamp value {simplified_expr:?}" + ))) } fn collect_tokens(w: Token, parser: &mut Parser) -> Vec { From cd8b958683f47030de90bd8c6ab7818563172d1c Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 14 Mar 2024 10:00:44 +0000 Subject: [PATCH 03/16] fix(tql): address taplo issue --- src/sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml index b3ae8155b4e4..8be9eef6696b 100644 --- a/src/sql/Cargo.toml +++ b/src/sql/Cargo.toml @@ -17,11 +17,11 @@ common-error.workspace = true common-macro.workspace = true common-query.workspace = true common-time.workspace = true +datafusion.workspace = true datafusion-common.workspace = true datafusion-expr.workspace = true datafusion-physical-expr.workspace = true datafusion-sql.workspace = true -datafusion.workspace = true datatypes.workspace = true hex = "0.4" itertools.workspace = true From 77cfa9499f181492327f144a30273e750c5ebf0e Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 14 Mar 2024 20:56:19 +0000 Subject: [PATCH 04/16] feat(tql): update parse_tql_query logic --- src/sql/src/parsers/tql_parser.rs | 88 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index a353d8c0db25..f2c153dd57be 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -177,55 +177,30 @@ impl<'a> ParserContext<'a> { tokens } + pub fn is_word_or_eof(parser: &mut Parser) -> bool { + matches!(parser.peek_token().token, Token::Word(_) | Token::EOF) + } + fn parse_tql_query( parser: &mut Parser, sql: &str, - delimiter: &str, + _delimiter: &str, ) -> std::result::Result { - let index = sql.to_uppercase().find(delimiter); - - if let Some(index) = index { - let index = index + delimiter.len() + 1; - if index >= sql.len() { - return Err(ParserError::ParserError("empty TQL query".to_string())); - } - - let query = if delimiter == ")" { - match Self::find_last_balanced_bracket(sql) { - Some(to) => &sql[(to + 1)..], - None => &sql[index..], - } - } else { - &sql[index..] - }; - - while parser.next_token() != Token::EOF { - // consume all tokens - // TODO(dennis): supports multi TQL statements separated by ';'? - } - - // remove the last ';' or tailing space if exists - Ok(query.trim().trim_end_matches(';').to_string()) - } else { - Err(ParserError::ParserError(format!("{delimiter} not found",))) + while !Self::is_word_or_eof(parser) { + let _ = parser.next_token(); + } + let index = parser.next_token().location.column as usize; + if index == 0 { + return Err(ParserError::ParserError("empty TQL query".to_string())); } - } - fn find_last_balanced_bracket(sql: &str) -> Option { - let mut balance = 0; - for (index, c) in sql.char_indices() { - match c { - '(' => balance += 1, - ')' => { - balance -= 1; - if balance == 0 { - return Some(index); - } - } - _ => {} - } + let query = &sql[index - 1..]; + while parser.next_token() != Token::EOF { + // consume all tokens + // TODO(dennis): supports multi TQL statements separated by ';'? } - None + // remove the last ';' or tailing space if exists + Ok(query.trim().trim_end_matches(';').to_string()) } fn parse_tql_explain(&mut self) -> Result { @@ -325,6 +300,7 @@ mod tests { #[test] fn test_parse_tql_eval_with_functions() { let sql = "TQL EVAL (now() - '10 minutes'::interval, now(), '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + let mut result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); @@ -336,7 +312,6 @@ mod tests { .unwrap_or(0); let statement = result.remove(0); - match statement { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, (assert_time - 600).to_string()); @@ -531,6 +506,26 @@ mod tests { } } + #[test] + fn test_parse_tql_with_whitespaces() { + let sql = "TQL EVAL (0, 30, '10s') , data + (1 < bool 2);"; + let mut result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + assert_eq!(1, result.len()); + + let statement = result.remove(0); + match statement { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "0"); + assert_eq!(eval.end, "30"); + assert_eq!(eval.step, "10s"); + assert_eq!(eval.query, "data + (1 < bool 2)"); + } + _ => unreachable!(), + } + } + #[test] fn test_parse_tql_error() { // Invalid duration @@ -546,5 +541,12 @@ mod tests { ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap_err(); assert!(result.output_msg().contains("Expected ,, found: )")); + + // empty TQL query + let sql = "TQL EVAL (0, 30, '10s')"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap_err(); + assert!(result.output_msg().contains("empty TQL query")); } } From 4d8711e730b3ca4b90dfa8113ffb8fd8ae11d750 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Fri, 15 Mar 2024 15:51:28 +0000 Subject: [PATCH 05/16] fix(tql): change query parsing logic to use parser instead of delimiter --- src/sql/src/parsers/tql_parser.rs | 272 +++++++++++++----------------- 1 file changed, 115 insertions(+), 157 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index f2c153dd57be..4c5a442deebb 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -34,7 +34,6 @@ use crate::statements::tql::{Tql, TqlAnalyze, TqlEval, TqlExplain}; pub const TQL: &str = "TQL"; const EVAL: &str = "EVAL"; const EVALUATE: &str = "EVALUATE"; -const EXPLAIN: &str = "EXPLAIN"; const VERBOSE: &str = "VERBOSE"; use datatypes::arrow::datatypes::DataType; @@ -53,23 +52,30 @@ impl<'a> ParserContext<'a> { match self.parser.peek_token().token { Token::Word(w) => { let uppercase = w.value.to_uppercase(); + let _consume_tql_token = self.parser.next_token(); match w.keyword { Keyword::NoKeyword if (uppercase == EVAL || uppercase == EVALUATE) && w.quote_style.is_none() => { - let _ = self.parser.next_token(); self.parse_tql_eval().context(error::SyntaxSnafu) } Keyword::EXPLAIN => { - let _ = self.parser.next_token(); - self.parse_tql_explain() + let is_verbose = self.peek_token_as_string() == VERBOSE; + if is_verbose { + let _consume_verbose_token = self.parser.next_token(); + } + self.parse_tql_explain(is_verbose) } Keyword::ANALYZE => { - let _ = self.parser.next_token(); - self.parse_tql_analyze().context(error::SyntaxSnafu) + let is_verbose = self.peek_token_as_string() == VERBOSE; + if is_verbose { + let _consume_verbose_token = self.parser.next_token(); + } + self.parse_tql_analyze(is_verbose) + .context(error::SyntaxSnafu) } _ => self.unsupported(self.peek_token_as_string()), } @@ -84,7 +90,7 @@ impl<'a> ParserContext<'a> { let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; - let query = Self::parse_tql_query(parser, self.sql, ")")?; + let query = Self::parse_tql_query(parser, self.sql)?; Ok(Statement::Tql(Tql::Eval(TqlEval { start, @@ -94,25 +100,6 @@ impl<'a> ParserContext<'a> { }))) } - fn parse_string_or_number( - parser: &mut Parser, - token: Token, - ) -> std::result::Result { - let value = match parser.next_token().token { - Token::Number(n, _) => n, - Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, - unexpected => { - return Err(ParserError::ParserError(format!( - "Expect number or string, but is {unexpected:?}" - ))); - } - }; - parser.expect_token(&token)?; - - Ok(value) - } - - // TODO this method will substitute `parse_string_or_number` eventually fn parse_string_or_number_or_word( parser: &mut Parser, token: Token, @@ -177,16 +164,16 @@ impl<'a> ParserContext<'a> { tokens } - pub fn is_word_or_eof(parser: &mut Parser) -> bool { - matches!(parser.peek_token().token, Token::Word(_) | Token::EOF) + /// Seeks to the start of a query by moving the parser + /// to the first meaningful token, skipping over any leading commas tokens. + /// Returns `true` if the parser successfully moved to the first meaningful token, + /// otherwise returns `false` + pub fn seek_to_query_start(parser: &mut Parser) -> bool { + !matches!(parser.peek_token().token, Token::Comma) } - fn parse_tql_query( - parser: &mut Parser, - sql: &str, - _delimiter: &str, - ) -> std::result::Result { - while !Self::is_word_or_eof(parser) { + fn parse_tql_query(parser: &mut Parser, sql: &str) -> std::result::Result { + while !Self::seek_to_query_start(parser) { let _ = parser.next_token(); } let index = parser.next_token().location.column as usize; @@ -203,29 +190,24 @@ impl<'a> ParserContext<'a> { Ok(query.trim().trim_end_matches(';').to_string()) } - fn parse_tql_explain(&mut self) -> Result { + fn parse_tql_explain(&mut self, is_verbose: bool) -> Result { let parser = &mut self.parser; - let is_verbose = if parser.peek_token().token.to_string() == VERBOSE { - let _ = parser.next_token(); - true - } else { - false - }; - let delimiter = match parser.expect_token(&Token::LParen) { - Ok(_) => ")", - Err(_) => { - if is_verbose { - VERBOSE - } else { - EXPLAIN - } + + let (start, end, step) = match parser.peek_token().token { + Token::LParen => { + let _consume_lparen_token = parser.next_token(); + let start = Self::parse_string_or_number_or_word(parser, Token::Comma) + .unwrap_or("0".to_string()); + let end = Self::parse_string_or_number_or_word(parser, Token::Comma) + .unwrap_or("0".to_string()); + let step = Self::parse_string_or_number_or_word(parser, Token::RParen) + .unwrap_or("5m".to_string()); + (start, end, step) } + _ => ("0".to_string(), "0".to_string(), "5m".to_string()), }; - let start = Self::parse_string_or_number(parser, Token::Comma).unwrap_or("0".to_string()); - let end = Self::parse_string_or_number(parser, Token::Comma).unwrap_or("0".to_string()); - let step = Self::parse_string_or_number(parser, Token::RParen).unwrap_or("5m".to_string()); - let query = - Self::parse_tql_query(parser, self.sql, delimiter).context(error::SyntaxSnafu)?; + + let query = Self::parse_tql_query(parser, self.sql).context(error::SyntaxSnafu)?; Ok(Statement::Tql(Tql::Explain(TqlExplain { query, @@ -236,20 +218,17 @@ impl<'a> ParserContext<'a> { }))) } - fn parse_tql_analyze(&mut self) -> std::result::Result { + fn parse_tql_analyze( + &mut self, + is_verbose: bool, + ) -> std::result::Result { let parser = &mut self.parser; - let is_verbose = if parser.peek_token().token.to_string() == VERBOSE { - let _ = parser.next_token(); - true - } else { - false - }; parser.expect_token(&Token::LParen)?; - let start = Self::parse_string_or_number(parser, Token::Comma)?; - let end = Self::parse_string_or_number(parser, Token::Comma)?; - let step = Self::parse_string_or_number(parser, Token::RParen)?; - let query = Self::parse_tql_query(parser, self.sql, ")")?; + let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; + let query = Self::parse_tql_query(parser, self.sql)?; Ok(Statement::Tql(Tql::Analyze(TqlAnalyze { start, end, @@ -297,22 +276,25 @@ mod tests { use crate::dialect::GreptimeDbDialect; use crate::parser::ParseOptions; - #[test] - fn test_parse_tql_eval_with_functions() { - let sql = "TQL EVAL (now() - '10 minutes'::interval, now(), '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - + fn parse_into_statement(sql: &str) -> Statement { let mut result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); assert_eq!(1, result.len()); + result.remove(0) + } + #[test] + fn test_parse_tql_eval_with_functions() { + let sql = "TQL EVAL (now() - '10 minutes'::interval, now(), '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; let assert_time = Utc::now() .timestamp_nanos_opt() .map(|x| x / 1_000_000_000) .unwrap_or(0); - - let statement = result.remove(0); - match statement { + // TODO `Utc::now()` introduces the flakiness in a test, + // left: "1710514410" + // right: "1710514409" + match parse_into_statement(sql) { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, (assert_time - 600).to_string()); assert_eq!(eval.end, assert_time.to_string()); @@ -326,14 +308,7 @@ mod tests { #[test] fn test_parse_tql_eval() { let sql = "TQL EVAL (1676887657, 1676887659, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, "1676887657"); assert_eq!(eval.end, "1676887659"); @@ -344,14 +319,9 @@ mod tests { } let sql = "TQL EVAL (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + let statement = parse_into_statement(sql); - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement.clone() { + match &statement { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, "1676887657.1"); assert_eq!(eval.end, "1676887659.5"); @@ -361,25 +331,12 @@ mod tests { _ => unreachable!(), } - let sql = "TQL EVALUATE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement2 = result.remove(0); + let sql2 = "TQL EVALUATE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + let statement2 = parse_into_statement(sql2); assert_eq!(statement, statement2); let sql = "tql eval ('2015-07-01T20:10:30.781Z', '2015-07-01T20:11:00.781Z', '30s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, "2015-07-01T20:10:30.781Z"); assert_eq!(eval.end, "2015-07-01T20:11:00.781Z"); @@ -393,14 +350,7 @@ mod tests { #[test] fn test_parse_tql_explain() { let sql = "TQL EXPLAIN http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert_eq!(explain.start, "0"); @@ -412,14 +362,7 @@ mod tests { } let sql = "TQL EXPLAIN VERBOSE http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert_eq!(explain.start, "0"); @@ -431,14 +374,7 @@ mod tests { } let sql = "TQL EXPLAIN (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert_eq!(explain.start, "20"); @@ -450,14 +386,7 @@ mod tests { } let sql = "TQL EXPLAIN VERBOSE (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert_eq!(explain.start, "20"); @@ -472,12 +401,7 @@ mod tests { #[test] fn test_parse_tql_analyze() { let sql = "TQL ANALYZE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Analyze(analyze)) => { assert_eq!(analyze.start, "1676887657.1"); assert_eq!(analyze.end, "1676887659.5"); @@ -489,12 +413,7 @@ mod tests { } let sql = "TQL ANALYZE VERBOSE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - let statement = result.remove(0); - match statement { + match parse_into_statement(sql) { Statement::Tql(Tql::Analyze(analyze)) => { assert_eq!(analyze.start, "1676887657.1"); assert_eq!(analyze.end, "1676887659.5"); @@ -507,15 +426,10 @@ mod tests { } #[test] - fn test_parse_tql_with_whitespaces() { - let sql = "TQL EVAL (0, 30, '10s') , data + (1 < bool 2);"; - let mut result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - assert_eq!(1, result.len()); - - let statement = result.remove(0); - match statement { + fn test_parse_tql_with_various_queries() { + // query has whitespaces and comma + match parse_into_statement("TQL EVAL (0, 30, '10s') , data + (1 < bool 2);") + { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, "0"); assert_eq!(eval.end, "30"); @@ -524,11 +438,55 @@ mod tests { } _ => unreachable!(), } + // query starts with a quote + match parse_into_statement("TQL EVAL (0, 10, '5s') '1+1';") { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "0"); + assert_eq!(eval.end, "10"); + assert_eq!(eval.step, "5s"); + assert_eq!(eval.query, "'1+1'"); + } + _ => unreachable!(), + } + + // query starts with number + match parse_into_statement("TQL EVAL (300, 300, '1s') 10 atan2 20;") { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "300"); + assert_eq!(eval.end, "300"); + assert_eq!(eval.step, "1s"); + assert_eq!(eval.query, "10 atan2 20"); + } + _ => unreachable!(), + } + + // query starts with a bracket + let sql = "TQL EVAL (0, 30, '10s') (sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100;"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "0"); + assert_eq!(eval.end, "30"); + assert_eq!(eval.step, "10s"); + assert_eq!(eval.query, "(sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100"); + } + _ => unreachable!(), + } + + // query starts with a curly bracket + match parse_into_statement("TQL EVAL (0, 10, '5s') {__name__=\"test\"}") { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "0"); + assert_eq!(eval.end, "10"); + assert_eq!(eval.step, "5s"); + assert_eq!(eval.query, "{__name__=\"test\"}"); + } + _ => unreachable!(), + } } #[test] fn test_parse_tql_error() { - // Invalid duration + // invalid duration let sql = "TQL EVAL (1676887657, 1676887659, 1m) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) From b665614dd2a49e78480e6358459f7a36e25fa165 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Sun, 17 Mar 2024 14:49:46 +0000 Subject: [PATCH 06/16] fix(tql): add timestamp function support, add sqlness tests --- Cargo.lock | 5 + src/sql/src/parsers/tql_parser.rs | 106 ++++++++++-------- .../cases/standalone/common/tql/basic.result | 22 ++++ tests/cases/standalone/common/tql/basic.sql | 4 + 4 files changed, 92 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8ca9d963bdf..fa38e03926e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9529,6 +9529,7 @@ name = "sql" version = "0.7.0" dependencies = [ "api", + "chrono", "common-base", "common-catalog", "common-datasource", @@ -9537,6 +9538,10 @@ dependencies = [ "common-macro", "common-query", "common-time", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-sql", "datatypes", "hex", diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 4c5a442deebb..4b56d51b1b1c 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -52,7 +52,7 @@ impl<'a> ParserContext<'a> { match self.parser.peek_token().token { Token::Word(w) => { let uppercase = w.value.to_uppercase(); - let _consume_tql_token = self.parser.next_token(); + let _consume_tql_keyword_token = self.parser.next_token(); match w.keyword { Keyword::NoKeyword if (uppercase == EVAL || uppercase == EVALUATE) @@ -100,29 +100,46 @@ impl<'a> ParserContext<'a> { }))) } + pub fn peek_to_token(parser: &mut Parser, token: &Token) -> bool { + match parser.peek_token().token { + Token::Comma => matches!(token, Token::Comma), + Token::RParen => matches!(token, Token::RParen), + _ => false, + } + } + fn parse_string_or_number_or_word( parser: &mut Parser, - token: Token, + delimiter_token: Token, ) -> std::result::Result { - let value = match parser.next_token().token { - Token::Number(n, _) => n, - Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, - Token::Word(w) => Self::parse_tql_word(Token::Word(w), parser)?, - unexpected => { - return Err(ParserError::ParserError(format!( - "Expect number, string or word, but is {unexpected:?}" - ))); + let mut tokens = vec![]; + while !Self::peek_to_token(parser, &delimiter_token) { + let token = parser.next_token(); + tokens.push(token.token); + } + let result = match tokens.len() { + 0 => Err(ParserError::ParserError("Expected tokens".to_string())), + 1 => { + let value = match tokens[0].clone() { + Token::Number(n, _) => n, + Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, + Token::Word(_) => Self::parse_tql_word(tokens)?, + unexpected => { + return Err(ParserError::ParserError(format!( + "Expect number, string or word, but is {unexpected:?}" + ))); + } + }; + Ok(value) } + _ => Self::parse_tql_word(tokens), }; - parser.expect_token(&token)?; - Ok(value) + parser.expect_token(&delimiter_token)?; + result } - fn parse_tql_word(w: Token, parser: &mut Parser) -> std::result::Result { - let tokens = Self::collect_tokens(w, parser); + fn parse_tql_word(tokens: Vec) -> std::result::Result { let empty_df_schema = DFSchema::empty(); - let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); - let expr = Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) .parse_expr() @@ -137,6 +154,7 @@ impl<'a> ParserContext<'a> { ParserError::ParserError(format!("Failed to convert to logical expression {err:?}")) })?; + let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); let simplified_expr = ExprSimplifier::new(info) .simplify(logical_expr) @@ -154,27 +172,9 @@ impl<'a> ParserContext<'a> { ))) } - fn collect_tokens(w: Token, parser: &mut Parser) -> Vec { - let mut tokens = vec![]; - tokens.push(w); - while parser.peek_token() != Token::Comma { - let token = parser.next_token(); - tokens.push(token.token); - } - tokens - } - - /// Seeks to the start of a query by moving the parser - /// to the first meaningful token, skipping over any leading commas tokens. - /// Returns `true` if the parser successfully moved to the first meaningful token, - /// otherwise returns `false` - pub fn seek_to_query_start(parser: &mut Parser) -> bool { - !matches!(parser.peek_token().token, Token::Comma) - } - fn parse_tql_query(parser: &mut Parser, sql: &str) -> std::result::Result { - while !Self::seek_to_query_start(parser) { - let _ = parser.next_token(); + while matches!(parser.peek_token().token, Token::Comma) { + let _skip_token = parser.next_token(); } let index = parser.next_token().location.column as usize; if index == 0 { @@ -294,7 +294,8 @@ mod tests { // TODO `Utc::now()` introduces the flakiness in a test, // left: "1710514410" // right: "1710514409" - match parse_into_statement(sql) { + let statement = parse_into_statement(sql); + match statement { Statement::Tql(Tql::Eval(eval)) => { assert_eq!(eval.start, (assert_time - 600).to_string()); assert_eq!(eval.end, assert_time.to_string()); @@ -303,6 +304,17 @@ mod tests { } _ => unreachable!(), } + + let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "300"); + assert_eq!(eval.end, "1200"); + assert_eq!(eval.step, "1m"); + assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + } + _ => unreachable!(), + } } #[test] @@ -486,25 +498,29 @@ mod tests { #[test] fn test_parse_tql_error() { + let dialect = &GreptimeDbDialect {}; + let parse_options = ParseOptions::default(); + // invalid duration let sql = "TQL EVAL (1676887657, 1676887659, 1m) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; let result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap_err(); - assert!(result.output_msg().contains("Expected ), found: m")); + ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err(); + assert!(result + .output_msg() + .contains("Failed to extract a timestamp value")); // missing end let sql = "TQL EVAL (1676887657, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; let result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap_err(); - assert!(result.output_msg().contains("Expected ,, found: )")); + ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err(); + assert!(result + .output_msg() + .contains("Failed to extract a timestamp value")); // empty TQL query let sql = "TQL EVAL (0, 30, '10s')"; let result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap_err(); + ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err(); assert!(result.output_msg().contains("empty TQL query")); } } diff --git a/tests/cases/standalone/common/tql/basic.result b/tests/cases/standalone/common/tql/basic.result index 8d0229dd3164..1af7ae94cc7c 100644 --- a/tests/cases/standalone/common/tql/basic.result +++ b/tests/cases/standalone/common/tql/basic.result @@ -59,6 +59,28 @@ TQL EVAL (0, 10, '5s') test{k="a"}; | 2.0 | 1970-01-01T00:00:10 | a | +-----+---------------------+---+ +TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '1s') test{k="a"}; + ++-----+---------------------+---+ +| i | j | k | ++-----+---------------------+---+ +| 2.0 | 1970-01-01T00:00:01 | a | +| 2.0 | 1970-01-01T00:00:02 | a | +| 2.0 | 1970-01-01T00:00:03 | a | +| 2.0 | 1970-01-01T00:00:04 | a | +| 2.0 | 1970-01-01T00:00:05 | a | +| 2.0 | 1970-01-01T00:00:06 | a | +| 2.0 | 1970-01-01T00:00:07 | a | +| 2.0 | 1970-01-01T00:00:08 | a | +| 2.0 | 1970-01-01T00:00:09 | a | +| 2.0 | 1970-01-01T00:00:10 | a | ++-----+---------------------+---+ + +TQL EVAL (now() - '2 minutes'::interval, now(), '5s') test{k="a"}; + +++ +++ + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql/basic.sql b/tests/cases/standalone/common/tql/basic.sql index bd21518665f7..ec8cc890bc77 100644 --- a/tests/cases/standalone/common/tql/basic.sql +++ b/tests/cases/standalone/common/tql/basic.sql @@ -19,4 +19,8 @@ TQL EVAL (0, 10, '5s') {__name__!="test"}; -- the point at 1ms will be shadowed by the point at 2ms TQL EVAL (0, 10, '5s') test{k="a"}; +TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '1s') test{k="a"}; + +TQL EVAL (now() - '2 minutes'::interval, now(), '5s') test{k="a"}; + DROP TABLE test; From 26a4ab2337ad58427412f462736eaa0028f98f94 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Mon, 18 Mar 2024 15:27:59 +0000 Subject: [PATCH 07/16] fix(tql): add lookback optional param for tql eval --- src/sql/src/parsers/tql_parser.rs | 80 +++++++++++++++++++++++++++---- src/sql/src/statements/tql.rs | 2 + 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 4b56d51b1b1c..fed2c76412b6 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -89,7 +89,15 @@ impl<'a> ParserContext<'a> { parser.expect_token(&Token::LParen)?; let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; + let delimiter_token = Self::find_next_delimiter_token(parser); + let (step, lookback) = if Self::is_comma(&delimiter_token) { + let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let lookback = Self::parse_string_or_number_or_word(parser, Token::RParen).ok(); + (step, lookback) + } else { + let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; + (step, None) + }; let query = Self::parse_tql_query(parser, self.sql)?; Ok(Statement::Tql(Tql::Eval(TqlEval { @@ -97,23 +105,45 @@ impl<'a> ParserContext<'a> { end, step, query, + lookback, }))) } - pub fn peek_to_token(parser: &mut Parser, token: &Token) -> bool { - match parser.peek_token().token { - Token::Comma => matches!(token, Token::Comma), - Token::RParen => matches!(token, Token::RParen), + fn find_next_delimiter_token(parser: &mut Parser) -> Token { + let mut n: usize = 0; + while !(Self::is_comma(&parser.peek_nth_token(n).token) + || Self::is_rparen(&parser.peek_nth_token(n).token)) + { + n += 1; + } + parser.peek_nth_token(n).token + } + + pub fn is_delimiter_token(token: &Token, delimiter_token: &Token) -> bool { + match token { + Token::Comma => Self::is_comma(delimiter_token), + Token::RParen => Self::is_rparen(delimiter_token), _ => false, } } + #[inline] + fn is_comma(token: &Token) -> bool { + matches!(token, Token::Comma) + } + + #[inline] + fn is_rparen(token: &Token) -> bool { + matches!(token, Token::RParen) + } + fn parse_string_or_number_or_word( parser: &mut Parser, delimiter_token: Token, ) -> std::result::Result { let mut tokens = vec![]; - while !Self::peek_to_token(parser, &delimiter_token) { + + while !Self::is_delimiter_token(&parser.peek_token().token, &delimiter_token) { let token = parser.next_token(); tokens.push(token.token); } @@ -123,7 +153,7 @@ impl<'a> ParserContext<'a> { let value = match tokens[0].clone() { Token::Number(n, _) => n, Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, - Token::Word(_) => Self::parse_tql_word(tokens)?, + Token::Word(_) => Self::parse_tokens(tokens)?, unexpected => { return Err(ParserError::ParserError(format!( "Expect number, string or word, but is {unexpected:?}" @@ -132,13 +162,13 @@ impl<'a> ParserContext<'a> { }; Ok(value) } - _ => Self::parse_tql_word(tokens), + _ => Self::parse_tokens(tokens), }; parser.expect_token(&delimiter_token)?; result } - fn parse_tql_word(tokens: Vec) -> std::result::Result { + fn parse_tokens(tokens: Vec) -> std::result::Result { let empty_df_schema = DFSchema::empty(); let expr = Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) @@ -300,6 +330,7 @@ mod tests { assert_eq!(eval.start, (assert_time - 600).to_string()); assert_eq!(eval.end, assert_time.to_string()); assert_eq!(eval.step, "1m"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } _ => unreachable!(), @@ -311,6 +342,7 @@ mod tests { assert_eq!(eval.start, "300"); assert_eq!(eval.end, "1200"); assert_eq!(eval.step, "1m"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } _ => unreachable!(), @@ -325,6 +357,7 @@ mod tests { assert_eq!(eval.start, "1676887657"); assert_eq!(eval.end, "1676887659"); assert_eq!(eval.step, "1m"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } _ => unreachable!(), @@ -338,6 +371,7 @@ mod tests { assert_eq!(eval.start, "1676887657.1"); assert_eq!(eval.end, "1676887659.5"); assert_eq!(eval.step, "30.3"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } _ => unreachable!(), @@ -353,6 +387,34 @@ mod tests { assert_eq!(eval.start, "2015-07-01T20:10:30.781Z"); assert_eq!(eval.end, "2015-07-01T20:11:00.781Z"); assert_eq!(eval.step, "30s"); + assert_eq!(eval.lookback, None); + assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + } + _ => unreachable!(), + } + } + + #[test] + fn test_parse_tql_eval_with_lookback_values() { + let sql = "TQL EVAL (1676887657, 1676887659, '1m', '5m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "1676887657"); + assert_eq!(eval.end, "1676887659"); + assert_eq!(eval.step, "1m".to_string()); + assert_eq!(eval.lookback, Some("5m".to_string())); + assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + } + _ => unreachable!(), + } + + let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m', '7m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Eval(eval)) => { + assert_eq!(eval.start, "300"); + assert_eq!(eval.end, "1200"); + assert_eq!(eval.step, "1m"); + assert_eq!(eval.lookback, Some("7m".to_string())); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } _ => unreachable!(), diff --git a/src/sql/src/statements/tql.rs b/src/sql/src/statements/tql.rs index 46ca696f82ab..a87f9c71c08e 100644 --- a/src/sql/src/statements/tql.rs +++ b/src/sql/src/statements/tql.rs @@ -21,12 +21,14 @@ pub enum Tql { Analyze(TqlAnalyze), } +/// TQL EVAL (, , , [lookback]) #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)] pub struct TqlEval { pub start: String, pub end: String, pub step: String, pub query: String, + pub lookback: Option, } /// TQL EXPLAIN [VERBOSE] (like SQL EXPLAIN): doesn't execute the query but tells how the query would be executed. From 4bff8a51e1dba9862a5351764e76f5d96d81ef7c Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 20 Mar 2024 15:27:07 +0000 Subject: [PATCH 08/16] fix(tql): adjust tests for now() function --- src/sql/src/parsers/tql_parser.rs | 25 +++++++------------ .../cases/standalone/common/tql/basic.result | 18 ++++++++++--- tests/cases/standalone/common/tql/basic.sql | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index fed2c76412b6..1179d63bae38 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -176,8 +176,7 @@ impl<'a> ParserContext<'a> { .map_err(|err| { ParserError::ParserError(format!("Failed to convert to expression: {err:?}")) })?; - - let sql_to_rel = SqlToRel::new(&DummyContextProvider {}); + let sql_to_rel = SqlToRel::new(&StubContextProvider {}); let logical_expr = sql_to_rel .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) .map_err(|err| { @@ -193,7 +192,8 @@ impl<'a> ParserContext<'a> { })?; match simplified_expr { - Expr::Literal(ScalarValue::TimestampNanosecond(v, _)) => v, + Expr::Literal(ScalarValue::TimestampNanosecond(v, _)) + | Expr::Literal(ScalarValue::DurationNanosecond(v)) => v, _ => None, } .map(|timestamp_nanos| (timestamp_nanos / 1_000_000_000).to_string()) @@ -270,9 +270,9 @@ impl<'a> ParserContext<'a> { } #[derive(Default)] -struct DummyContextProvider {} +struct StubContextProvider {} -impl ContextProvider for DummyContextProvider { +impl ContextProvider for StubContextProvider { fn get_table_provider(&self, _name: TableReference) -> DFResult> { unimplemented!() } @@ -316,20 +316,13 @@ mod tests { #[test] fn test_parse_tql_eval_with_functions() { - let sql = "TQL EVAL (now() - '10 minutes'::interval, now(), '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; - let assert_time = Utc::now() - .timestamp_nanos_opt() - .map(|x| x / 1_000_000_000) - .unwrap_or(0); - // TODO `Utc::now()` introduces the flakiness in a test, - // left: "1710514410" - // right: "1710514409" + let sql = "TQL EVAL (now() - now(), now() - (now() - '10 seconds'::interval), '1s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; let statement = parse_into_statement(sql); match statement { Statement::Tql(Tql::Eval(eval)) => { - assert_eq!(eval.start, (assert_time - 600).to_string()); - assert_eq!(eval.end, assert_time.to_string()); - assert_eq!(eval.step, "1m"); + assert_eq!(eval.start, "0"); + assert_eq!(eval.end, "10"); + assert_eq!(eval.step, "1s"); assert_eq!(eval.lookback, None); assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); } diff --git a/tests/cases/standalone/common/tql/basic.result b/tests/cases/standalone/common/tql/basic.result index 1af7ae94cc7c..50e71aa1e4b8 100644 --- a/tests/cases/standalone/common/tql/basic.result +++ b/tests/cases/standalone/common/tql/basic.result @@ -76,10 +76,22 @@ TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + ' | 2.0 | 1970-01-01T00:00:10 | a | +-----+---------------------+---+ -TQL EVAL (now() - '2 minutes'::interval, now(), '5s') test{k="a"}; +TQL EVAL (now() - now(), now() - (now() - '10 seconds'::interval), '1s') test{k="a"}; -++ -++ ++-----+---------------------+---+ +| i | j | k | ++-----+---------------------+---+ +| 2.0 | 1970-01-01T00:00:01 | a | +| 2.0 | 1970-01-01T00:00:02 | a | +| 2.0 | 1970-01-01T00:00:03 | a | +| 2.0 | 1970-01-01T00:00:04 | a | +| 2.0 | 1970-01-01T00:00:05 | a | +| 2.0 | 1970-01-01T00:00:06 | a | +| 2.0 | 1970-01-01T00:00:07 | a | +| 2.0 | 1970-01-01T00:00:08 | a | +| 2.0 | 1970-01-01T00:00:09 | a | +| 2.0 | 1970-01-01T00:00:10 | a | ++-----+---------------------+---+ DROP TABLE test; diff --git a/tests/cases/standalone/common/tql/basic.sql b/tests/cases/standalone/common/tql/basic.sql index ec8cc890bc77..db9ff597832f 100644 --- a/tests/cases/standalone/common/tql/basic.sql +++ b/tests/cases/standalone/common/tql/basic.sql @@ -21,6 +21,6 @@ TQL EVAL (0, 10, '5s') test{k="a"}; TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '1s') test{k="a"}; -TQL EVAL (now() - '2 minutes'::interval, now(), '5s') test{k="a"}; +TQL EVAL (now() - now(), now() - (now() - '10 seconds'::interval), '1s') test{k="a"}; DROP TABLE test; From d91c4e96115d09c5cb4e20cc82c319ac8d3fbf52 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 21 Mar 2024 11:49:51 +0000 Subject: [PATCH 09/16] fix(tql): introduce the tqlerror to differentiate failures on parsing, evaluation and simplification stages --- src/sql/src/error.rs | 2 +- src/sql/src/parsers/tql_parser.rs | 90 ++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs index 24643b4a6505..b85c164a1d99 100644 --- a/src/sql/src/error.rs +++ b/src/sql/src/error.rs @@ -28,7 +28,7 @@ use crate::ast::{Expr, Value as SqlValue}; pub type Result = std::result::Result; /// SQL parser errors. -// Now the error in parser does not contains backtrace to avoid generating backtrace +// Now the error in parser does not contain backtrace to avoid generating backtrace // every time the parser parses an invalid SQL. #[derive(Snafu)] #[snafu(visibility(pub))] diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 1179d63bae38..57d5f862c7d2 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -17,7 +17,9 @@ use std::sync::Arc; use chrono::Utc; use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext}; use datafusion_common::config::ConfigOptions; -use datafusion_common::{DFSchema, Result as DFResult, ScalarValue, TableReference}; +use datafusion_common::{ + DFSchema, DataFusionError, Result as DFResult, ScalarValue, TableReference, +}; use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; use datafusion_physical_expr::execution_props::ExecutionProps; use datafusion_sql::planner::{ContextProvider, SqlToRel}; @@ -41,6 +43,46 @@ use sqlparser::parser::Parser; use crate::dialect::GreptimeDbDialect; +pub enum TQLError { + Parser(String), + Simplification(String), + Evaluation(String), +} + +impl From for TQLError { + fn from(err: ParserError) -> Self { + TQLError::Parser(err.to_string()) + } +} + +impl From for TQLError { + fn from(err: DataFusionError) -> Self { + match err { + DataFusionError::SQL(parser_err) => TQLError::Parser(parser_err.to_string()), + DataFusionError::Plan(plan_err) => TQLError::Evaluation(plan_err), + unspecified => { + TQLError::Evaluation(format!("Failed to evaluate due to: {unspecified:?}")) + } + } + } +} + +impl From for ParserError { + fn from(tql_err: TQLError) -> Self { + match tql_err { + TQLError::Parser(s) => { + ParserError::ParserError(format!("Failed to parse the query: {s}")) + } + TQLError::Simplification(s) => { + ParserError::ParserError(format!("Failed to simplify the query: {s}")) + } + TQLError::Evaluation(s) => { + ParserError::ParserError(format!("Failed to evaluate the query: {s}")) + } + } + } +} + /// TQL extension parser, including: /// - `TQL EVAL ` /// - `TQL EXPLAIN [VERBOSE] ` @@ -140,7 +182,7 @@ impl<'a> ParserContext<'a> { fn parse_string_or_number_or_word( parser: &mut Parser, delimiter_token: Token, - ) -> std::result::Result { + ) -> std::result::Result { let mut tokens = vec![]; while !Self::is_delimiter_token(&parser.peek_token().token, &delimiter_token) { @@ -148,14 +190,14 @@ impl<'a> ParserContext<'a> { tokens.push(token.token); } let result = match tokens.len() { - 0 => Err(ParserError::ParserError("Expected tokens".to_string())), + 0 => Err(TQLError::Parser("Expected tokens".to_string())), 1 => { let value = match tokens[0].clone() { Token::Number(n, _) => n, Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, Token::Word(_) => Self::parse_tokens(tokens)?, unexpected => { - return Err(ParserError::ParserError(format!( + return Err(TQLError::Parser(format!( "Expect number, string or word, but is {unexpected:?}" ))); } @@ -168,36 +210,48 @@ impl<'a> ParserContext<'a> { result } - fn parse_tokens(tokens: Vec) -> std::result::Result { - let empty_df_schema = DFSchema::empty(); - let expr = Parser::new(&GreptimeDbDialect {}) + fn parse_tokens(tokens: Vec) -> std::result::Result { + Self::parse_to_expr(tokens) + .and_then(Self::parse_to_logical_expr) + .and_then(Self::simplify_expr) + .and_then(Self::evaluate_expr) + } + + fn parse_to_expr(tokens: Vec) -> std::result::Result { + Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) .parse_expr() - .map_err(|err| { - ParserError::ParserError(format!("Failed to convert to expression: {err:?}")) - })?; - let sql_to_rel = SqlToRel::new(&StubContextProvider {}); - let logical_expr = sql_to_rel + .map_err(|err| TQLError::Parser(format!("Failed to convert to expression: {err:?}"))) + } + + fn parse_to_logical_expr(expr: sqlparser::ast::Expr) -> std::result::Result { + let empty_df_schema = DFSchema::empty(); + SqlToRel::new(&StubContextProvider {}) .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) .map_err(|err| { - ParserError::ParserError(format!("Failed to convert to logical expression {err:?}")) - })?; + TQLError::Parser(format!("Failed to convert to logical expression {err:?}")) + }) + } + fn simplify_expr(logical_expr: Expr) -> std::result::Result { + let empty_df_schema = DFSchema::empty(); let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); - let simplified_expr = ExprSimplifier::new(info) + ExprSimplifier::new(info) .simplify(logical_expr) .map_err(|err| { - ParserError::ParserError(format!("Failed to simplify expression {err:?}")) - })?; + TQLError::Simplification(format!("Failed to simplify expression {err:?}")) + }) + } + fn evaluate_expr(simplified_expr: Expr) -> std::result::Result { match simplified_expr { Expr::Literal(ScalarValue::TimestampNanosecond(v, _)) | Expr::Literal(ScalarValue::DurationNanosecond(v)) => v, _ => None, } .map(|timestamp_nanos| (timestamp_nanos / 1_000_000_000).to_string()) - .ok_or(ParserError::ParserError(format!( + .ok_or(TQLError::Evaluation(format!( "Failed to extract a timestamp value {simplified_expr:?}" ))) } From b648e929c3f310cc468e2e7aec087e5cf7654df4 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 21 Mar 2024 13:43:53 +0000 Subject: [PATCH 10/16] fix(tql): add tests for explain/analyze --- src/sql/src/parsers/tql_parser.rs | 48 +++++++ .../common/tql-explain-analyze/analyze.result | 51 +++++++ .../common/tql-explain-analyze/analyze.sql | 18 +++ .../common/tql-explain-analyze/explain.result | 129 ++++++++++++++++++ .../common/tql-explain-analyze/explain.sql | 13 ++ 5 files changed, 259 insertions(+) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 57d5f862c7d2..18c2e4106b67 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -506,6 +506,18 @@ mod tests { _ => unreachable!(), } + let sql = "TQL EXPLAIN ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Explain(explain)) => { + assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert_eq!(explain.start, "300"); + assert_eq!(explain.end, "1200"); + assert_eq!(explain.step, "10"); + assert!(!explain.is_verbose); + } + _ => unreachable!(), + } + let sql = "TQL EXPLAIN VERBOSE (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { @@ -517,6 +529,18 @@ mod tests { } _ => unreachable!(), } + + let sql = "TQL EXPLAIN VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Explain(explain)) => { + assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert_eq!(explain.start, "300"); + assert_eq!(explain.end, "1200"); + assert_eq!(explain.step, "10"); + assert!(explain.is_verbose); + } + _ => unreachable!(), + } } #[test] @@ -533,6 +557,18 @@ mod tests { _ => unreachable!(), } + let sql = "TQL ANALYZE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Analyze(analyze)) => { + assert_eq!(analyze.start, "300"); + assert_eq!(analyze.end, "1200"); + assert_eq!(analyze.step, "10"); + assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert!(!analyze.is_verbose); + } + _ => unreachable!(), + } + let sql = "TQL ANALYZE VERBOSE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; match parse_into_statement(sql) { Statement::Tql(Tql::Analyze(analyze)) => { @@ -544,6 +580,18 @@ mod tests { } _ => unreachable!(), } + + let sql = "TQL ANALYZE VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Analyze(analyze)) => { + assert_eq!(analyze.start, "300"); + assert_eq!(analyze.end, "1200"); + assert_eq!(analyze.step, "10"); + assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert!(analyze.is_verbose); + } + _ => unreachable!(), + } } #[test] diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.result b/tests/cases/standalone/common/tql-explain-analyze/analyze.result index 9ac49f77f541..bccda79625bc 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.result @@ -27,6 +27,26 @@ TQL ANALYZE (0, 10, '5s') test; |_|_| +-+-+ +-- analyze at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED +|_|_PromSeriesDivideExec: tags=["k"], REDACTED +|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ + -- analyze verbose at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ @@ -58,6 +78,37 @@ TQL ANALYZE VERBOSE (0, 10, '5s') test; | REDACTED +-+-+ +-- analyze verbose at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (Duration.*) REDACTED +TQL ANALYZE VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| Plan with Metrics_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED +|_|_PromSeriesDivideExec: tags=["k"], REDACTED +|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED +|_|_MergeScanExec: REDACTED +|_|_| +| Plan with Full Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED +|_|_PromSeriesDivideExec: tags=["k"], REDACTED +|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED +|_|_MergeScanExec: REDACTED +|_|_| +| Output Rows_| 4_| +| REDACTED ++-+-+ + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql index 6fb8f3c0e555..84b14a76c302 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql @@ -11,6 +11,14 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED TQL ANALYZE (0, 10, '5s') test; +-- analyze at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + -- analyze verbose at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ @@ -21,4 +29,14 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (Duration.*) REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; +-- analyze verbose at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (Duration.*) REDACTED +TQL ANALYZE VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + DROP TABLE test; diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 3e1877654f2b..1403906ad4ff 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -28,6 +28,27 @@ TQL EXPLAIN (0, 10, '5s') test; | | | +---------------+-----------------------------------------------------------------------------------------------+ +-- explain at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + ++---------------+-----------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------+ +| logical_plan | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | +| | PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivide: tags=["k"] | +| | MergeScan [is_placeholder=false] | +| physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | +| | RepartitionExec: partitioning=REDACTED +| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@2 ASC NULLS LAST] | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------+ + -- explain verbose at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ @@ -136,6 +157,114 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_| +-+-+ +-- explain verbose at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +TQL EXPLAIN VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| initial_logical_plan_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_Sort: test.k DESC NULLS LAST, test.j DESC NULLS LAST_| +|_|_Filter: test.j >= TimestampMillisecond(-300000, None) AND test.j <= TimestampMillisecond(300000, None) | +|_|_TableScan: test_| +| logical_plan after count_wildcard_rule_| SAME TEXT AS ABOVE_| +| logical_plan after StringNormalizationRule_| SAME TEXT AS ABOVE_| +| logical_plan after inline_table_scan_| SAME TEXT AS ABOVE_| +| logical_plan after type_coercion_| SAME TEXT AS ABOVE_| +| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_MergeScan [is_placeholder=false]_| +| analyzed_logical_plan_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_nested_union_| SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after unwrap_cast_in_comparison_| SAME TEXT AS ABOVE_| +| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| +| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_| +| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| +| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after merge_projection_| SAME TEXT AS ABOVE_| +| logical_plan after rewrite_disjunctive_predicate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_| +| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_| +| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_one_union_| SAME TEXT AS ABOVE_| +| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_filter_| SAME TEXT AS ABOVE_| +| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_| +| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| +| logical_plan after unwrap_cast_in_comparison_| SAME TEXT AS ABOVE_| +| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_projection_| SAME TEXT AS ABOVE_| +| logical_plan after eliminate_projection_| SAME TEXT AS ABOVE_| +| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after OrderHintRule_| SAME TEXT AS ABOVE_| +| logical_plan_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivide: tags=["k"]_| +|_|_MergeScan [is_placeholder=false]_| +| initial_physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after OutputRequirements_| OutputRequirementExec_| +|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| +| physical_plan after join_selection_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceDistribution_| OutputRequirementExec_| +|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceSorting_| OutputRequirementExec_| +|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after coalesce_batches_| SAME TEXT AS ABOVE_| +| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| +|_|_MergeScanExec: REDACTED +|_|_| +| physical_plan after PipelineChecker_| SAME TEXT AS ABOVE_| +| physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| +| physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| +|_|_PromSeriesDivideExec: tags=["k"]_| +|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.sql b/tests/cases/standalone/common/tql-explain-analyze/explain.sql index 3b2c961933d3..b39c724e4f8e 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.sql @@ -8,6 +8,11 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED TQL EXPLAIN (0, 10, '5s') test; +-- explain at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + -- explain verbose at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ @@ -16,4 +21,12 @@ TQL EXPLAIN (0, 10, '5s') test; -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test; +-- explain verbose at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +TQL EXPLAIN VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; + DROP TABLE test; From 21f40aeb34439bc087f454c8c9b4aef79f480753 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Thu, 21 Mar 2024 16:40:14 +0000 Subject: [PATCH 11/16] feat(tql): add lookback support for explain/analyze, update tests --- src/sql/src/parsers/tql_parser.rs | 186 ++++++++++++++++++++++-------- src/sql/src/statements/tql.rs | 10 +- 2 files changed, 144 insertions(+), 52 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 18c2e4106b67..1a0efb48380e 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -109,6 +109,7 @@ impl<'a> ParserContext<'a> { let _consume_verbose_token = self.parser.next_token(); } self.parse_tql_explain(is_verbose) + .context(error::SyntaxSnafu) } Keyword::ANALYZE => { @@ -143,11 +144,79 @@ impl<'a> ParserContext<'a> { let query = Self::parse_tql_query(parser, self.sql)?; Ok(Statement::Tql(Tql::Eval(TqlEval { + start, + end, + step, + lookback, + query, + }))) + } + + fn parse_tql_explain( + &mut self, + is_verbose: bool, + ) -> std::result::Result { + let parser = &mut self.parser; + + let (start, end, step, lookback) = match parser.peek_token().token { + Token::LParen => { + let _consume_lparen_token = parser.next_token(); + let start = Self::parse_string_or_number_or_word(parser, Token::Comma) + .unwrap_or("0".to_string()); + let end = Self::parse_string_or_number_or_word(parser, Token::Comma) + .unwrap_or("0".to_string()); + let delimiter_token = Self::find_next_delimiter_token(parser); + let (step, lookback) = if Self::is_comma(&delimiter_token) { + let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let lookback = Self::parse_string_or_number_or_word(parser, Token::RParen).ok(); + (step, lookback) + } else { + let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; + (step, None) + }; + (start, end, step, lookback) + } + _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None), + }; + + let query = Self::parse_tql_query(parser, self.sql)?; + + Ok(Statement::Tql(Tql::Explain(TqlExplain { + query, + start, + end, + step, + lookback, + is_verbose, + }))) + } + + fn parse_tql_analyze( + &mut self, + is_verbose: bool, + ) -> std::result::Result { + let parser = &mut self.parser; + parser.expect_token(&Token::LParen)?; + let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let delimiter_token = Self::find_next_delimiter_token(parser); + let (step, lookback) = if Self::is_comma(&delimiter_token) { + let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let lookback = Self::parse_string_or_number_or_word(parser, Token::RParen).ok(); + (step, lookback) + } else { + let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; + (step, None) + }; + let query = Self::parse_tql_query(parser, self.sql)?; + + Ok(Statement::Tql(Tql::Analyze(TqlAnalyze { start, end, step, query, lookback, + is_verbose, }))) } @@ -273,54 +342,6 @@ impl<'a> ParserContext<'a> { // remove the last ';' or tailing space if exists Ok(query.trim().trim_end_matches(';').to_string()) } - - fn parse_tql_explain(&mut self, is_verbose: bool) -> Result { - let parser = &mut self.parser; - - let (start, end, step) = match parser.peek_token().token { - Token::LParen => { - let _consume_lparen_token = parser.next_token(); - let start = Self::parse_string_or_number_or_word(parser, Token::Comma) - .unwrap_or("0".to_string()); - let end = Self::parse_string_or_number_or_word(parser, Token::Comma) - .unwrap_or("0".to_string()); - let step = Self::parse_string_or_number_or_word(parser, Token::RParen) - .unwrap_or("5m".to_string()); - (start, end, step) - } - _ => ("0".to_string(), "0".to_string(), "5m".to_string()), - }; - - let query = Self::parse_tql_query(parser, self.sql).context(error::SyntaxSnafu)?; - - Ok(Statement::Tql(Tql::Explain(TqlExplain { - query, - start, - end, - step, - is_verbose, - }))) - } - - fn parse_tql_analyze( - &mut self, - is_verbose: bool, - ) -> std::result::Result { - let parser = &mut self.parser; - - parser.expect_token(&Token::LParen)?; - let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; - let query = Self::parse_tql_query(parser, self.sql)?; - Ok(Statement::Tql(Tql::Analyze(TqlAnalyze { - start, - end, - step, - query, - is_verbose, - }))) - } } #[derive(Default)] @@ -442,7 +463,7 @@ mod tests { } #[test] - fn test_parse_tql_eval_with_lookback_values() { + fn test_parse_tql_with_lookback_values() { let sql = "TQL EVAL (1676887657, 1676887659, '1m', '5m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; match parse_into_statement(sql) { Statement::Tql(Tql::Eval(eval)) => { @@ -466,6 +487,58 @@ mod tests { } _ => unreachable!(), } + + let sql = "TQL EXPLAIN (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Explain(explain)) => { + assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert_eq!(explain.start, "20"); + assert_eq!(explain.end, "100"); + assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, Some("3m".to_string())); + assert!(!explain.is_verbose); + } + _ => unreachable!(), + } + + let sql = "TQL EXPLAIN VERBOSE (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Explain(explain)) => { + assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert_eq!(explain.start, "20"); + assert_eq!(explain.end, "100"); + assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, Some("3m".to_string())); + assert!(explain.is_verbose); + } + _ => unreachable!(), + } + + let sql = "TQL ANALYZE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Analyze(analyze)) => { + assert_eq!(analyze.start, "1676887657"); + assert_eq!(analyze.end, "1676887659"); + assert_eq!(analyze.step, "1m"); + assert_eq!(analyze.lookback, Some("9m".to_string())); + assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert!(!analyze.is_verbose); + } + _ => unreachable!(), + } + + let sql = "TQL ANALYZE VERBOSE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Analyze(analyze)) => { + assert_eq!(analyze.start, "1676887657"); + assert_eq!(analyze.end, "1676887659"); + assert_eq!(analyze.step, "1m"); + assert_eq!(analyze.lookback, Some("9m".to_string())); + assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert!(analyze.is_verbose); + } + _ => unreachable!(), + } } #[test] @@ -477,6 +550,7 @@ mod tests { assert_eq!(explain.start, "0"); assert_eq!(explain.end, "0"); assert_eq!(explain.step, "5m"); + assert_eq!(explain.lookback, None); assert!(!explain.is_verbose); } _ => unreachable!(), @@ -489,6 +563,7 @@ mod tests { assert_eq!(explain.start, "0"); assert_eq!(explain.end, "0"); assert_eq!(explain.step, "5m"); + assert_eq!(explain.lookback, None); assert!(explain.is_verbose); } _ => unreachable!(), @@ -501,6 +576,7 @@ mod tests { assert_eq!(explain.start, "20"); assert_eq!(explain.end, "100"); assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, None); assert!(!explain.is_verbose); } _ => unreachable!(), @@ -513,6 +589,7 @@ mod tests { assert_eq!(explain.start, "300"); assert_eq!(explain.end, "1200"); assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, None); assert!(!explain.is_verbose); } _ => unreachable!(), @@ -525,6 +602,7 @@ mod tests { assert_eq!(explain.start, "20"); assert_eq!(explain.end, "100"); assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, None); assert!(explain.is_verbose); } _ => unreachable!(), @@ -537,6 +615,7 @@ mod tests { assert_eq!(explain.start, "300"); assert_eq!(explain.end, "1200"); assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, None); assert!(explain.is_verbose); } _ => unreachable!(), @@ -551,6 +630,7 @@ mod tests { assert_eq!(analyze.start, "1676887657.1"); assert_eq!(analyze.end, "1676887659.5"); assert_eq!(analyze.step, "30.3"); + assert_eq!(analyze.lookback, None); assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert!(!analyze.is_verbose); } @@ -563,6 +643,7 @@ mod tests { assert_eq!(analyze.start, "300"); assert_eq!(analyze.end, "1200"); assert_eq!(analyze.step, "10"); + assert_eq!(analyze.lookback, None); assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert!(!analyze.is_verbose); } @@ -575,6 +656,7 @@ mod tests { assert_eq!(analyze.start, "1676887657.1"); assert_eq!(analyze.end, "1676887659.5"); assert_eq!(analyze.step, "30.3"); + assert_eq!(analyze.lookback, None); assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert!(analyze.is_verbose); } @@ -587,6 +669,7 @@ mod tests { assert_eq!(analyze.start, "300"); assert_eq!(analyze.end, "1200"); assert_eq!(analyze.step, "10"); + assert_eq!(analyze.lookback, None); assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); assert!(analyze.is_verbose); } @@ -603,6 +686,7 @@ mod tests { assert_eq!(eval.start, "0"); assert_eq!(eval.end, "30"); assert_eq!(eval.step, "10s"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "data + (1 < bool 2)"); } _ => unreachable!(), @@ -613,6 +697,7 @@ mod tests { assert_eq!(eval.start, "0"); assert_eq!(eval.end, "10"); assert_eq!(eval.step, "5s"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "'1+1'"); } _ => unreachable!(), @@ -624,6 +709,7 @@ mod tests { assert_eq!(eval.start, "300"); assert_eq!(eval.end, "300"); assert_eq!(eval.step, "1s"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "10 atan2 20"); } _ => unreachable!(), @@ -636,6 +722,7 @@ mod tests { assert_eq!(eval.start, "0"); assert_eq!(eval.end, "30"); assert_eq!(eval.step, "10s"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "(sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100"); } _ => unreachable!(), @@ -647,6 +734,7 @@ mod tests { assert_eq!(eval.start, "0"); assert_eq!(eval.end, "10"); assert_eq!(eval.step, "5s"); + assert_eq!(eval.lookback, None); assert_eq!(eval.query, "{__name__=\"test\"}"); } _ => unreachable!(), diff --git a/src/sql/src/statements/tql.rs b/src/sql/src/statements/tql.rs index a87f9c71c08e..0261be880bed 100644 --- a/src/sql/src/statements/tql.rs +++ b/src/sql/src/statements/tql.rs @@ -27,26 +27,30 @@ pub struct TqlEval { pub start: String, pub end: String, pub step: String, - pub query: String, pub lookback: Option, + pub query: String, } -/// TQL EXPLAIN [VERBOSE] (like SQL EXPLAIN): doesn't execute the query but tells how the query would be executed. +/// TQL EXPLAIN [VERBOSE] [, , , [lookback]] +/// doesn't execute the query but tells how the query would be executed (similar to SQL EXPLAIN). #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)] pub struct TqlExplain { pub start: String, pub end: String, pub step: String, + pub lookback: Option, pub query: String, pub is_verbose: bool, } -/// TQL ANALYZE [VERBOSE] (like SQL ANALYZE): executes the plan and tells the detailed per-step execution time. +/// TQL ANALYZE [VERBOSE] (, , , [lookback]) +/// executes the plan and tells the detailed per-step execution time (similar to SQL ANALYZE). #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)] pub struct TqlAnalyze { pub start: String, pub end: String, pub step: String, + pub lookback: Option, pub query: String, pub is_verbose: bool, } From 47a36de07351004405a74b397da830bfd5491050 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Fri, 22 Mar 2024 09:59:15 +0000 Subject: [PATCH 12/16] feat(tql): add more sqlness tests --- .../common/tql-explain-analyze/analyze.result | 21 ++++++++++++++++++ .../common/tql-explain-analyze/analyze.sql | 9 ++++++++ .../common/tql-explain-analyze/explain.result | 22 +++++++++++++++++++ .../common/tql-explain-analyze/explain.sql | 6 +++++ .../cases/standalone/common/tql/basic.result | 18 +++++++++++++++ tests/cases/standalone/common/tql/basic.sql | 3 +++ 6 files changed, 79 insertions(+) diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.result b/tests/cases/standalone/common/tql-explain-analyze/analyze.result index bccda79625bc..a618e1e771b1 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.result @@ -27,6 +27,27 @@ TQL ANALYZE (0, 10, '5s') test; |_|_| +-+-+ +-- 'lookback' parameter is not fully supported, the test has to be updated +-- analyze at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +TQL ANALYZE (0, 10, '1s', '2s') test; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[1000], time index=[j], REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED +|_|_PromSeriesDivideExec: tags=["k"], REDACTED +|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ + -- analyze at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql index 84b14a76c302..8202bda665a9 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql @@ -11,6 +11,15 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED TQL ANALYZE (0, 10, '5s') test; +-- 'lookback' parameter is not fully supported, the test has to be updated +-- analyze at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +TQL ANALYZE (0, 10, '1s', '2s') test; + -- analyze at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 1403906ad4ff..8b97c6deb03b 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -28,6 +28,28 @@ TQL EXPLAIN (0, 10, '5s') test; | | | +---------------+-----------------------------------------------------------------------------------------------+ +-- 'lookback' parameter is not fully supported, the test has to be updated +-- explain at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +TQL EXPLAIN (0, 10, '1s', '2s') test; + ++---------------+-----------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------+ +| logical_plan | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | +| | PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivide: tags=["k"] | +| | MergeScan [is_placeholder=false] | +| physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | +| | RepartitionExec: partitioning=REDACTED +| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@2 ASC NULLS LAST] | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------+ + -- explain at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.sql b/tests/cases/standalone/common/tql-explain-analyze/explain.sql index b39c724e4f8e..aa0e25aaa4e8 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.sql @@ -8,6 +8,12 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED TQL EXPLAIN (0, 10, '5s') test; +-- 'lookback' parameter is not fully supported, the test has to be updated +-- explain at 0s, 5s and 10s. No point at 0s. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +TQL EXPLAIN (0, 10, '1s', '2s') test; + -- explain at 0s, 5s and 10s. No point at 0s. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED diff --git a/tests/cases/standalone/common/tql/basic.result b/tests/cases/standalone/common/tql/basic.result index 50e71aa1e4b8..f679dcb341d1 100644 --- a/tests/cases/standalone/common/tql/basic.result +++ b/tests/cases/standalone/common/tql/basic.result @@ -59,6 +59,24 @@ TQL EVAL (0, 10, '5s') test{k="a"}; | 2.0 | 1970-01-01T00:00:10 | a | +-----+---------------------+---+ +-- 'lookback' parameter is not fully supported, the test has to be updated +TQL EVAL (0, 10, '1s', '2s') test{k="a"}; + ++-----+---------------------+---+ +| i | j | k | ++-----+---------------------+---+ +| 2.0 | 1970-01-01T00:00:01 | a | +| 2.0 | 1970-01-01T00:00:02 | a | +| 2.0 | 1970-01-01T00:00:03 | a | +| 2.0 | 1970-01-01T00:00:04 | a | +| 2.0 | 1970-01-01T00:00:05 | a | +| 2.0 | 1970-01-01T00:00:06 | a | +| 2.0 | 1970-01-01T00:00:07 | a | +| 2.0 | 1970-01-01T00:00:08 | a | +| 2.0 | 1970-01-01T00:00:09 | a | +| 2.0 | 1970-01-01T00:00:10 | a | ++-----+---------------------+---+ + TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '1s') test{k="a"}; +-----+---------------------+---+ diff --git a/tests/cases/standalone/common/tql/basic.sql b/tests/cases/standalone/common/tql/basic.sql index db9ff597832f..9d9f3c8863b4 100644 --- a/tests/cases/standalone/common/tql/basic.sql +++ b/tests/cases/standalone/common/tql/basic.sql @@ -19,6 +19,9 @@ TQL EVAL (0, 10, '5s') {__name__!="test"}; -- the point at 1ms will be shadowed by the point at 2ms TQL EVAL (0, 10, '5s') test{k="a"}; +-- 'lookback' parameter is not fully supported, the test has to be updated +TQL EVAL (0, 10, '1s', '2s') test{k="a"}; + TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '1s') test{k="a"}; TQL EVAL (now() - now(), now() - (now() - '10 seconds'::interval), '1s') test{k="a"}; From bbac2f404341d24adc9ec929e7ab73680037d5d6 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Sun, 24 Mar 2024 08:28:41 +0000 Subject: [PATCH 13/16] chore(tql): extract common logic for eval, analyze and explain into a single function --- src/sql/src/parsers/tql_parser.rs | 86 +++++++------------------------ src/sql/src/statements/tql.rs | 67 ++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 1a0efb48380e..3fcf16de7dc0 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -31,7 +31,7 @@ use sqlparser::tokenizer::Token; use crate::error::{self, Result}; use crate::parser::ParserContext; use crate::statements::statement::Statement; -use crate::statements::tql::{Tql, TqlAnalyze, TqlEval, TqlExplain}; +use crate::statements::tql::{Tql, TqlAnalyze, TqlEval, TqlExplain, TqlParameters}; pub const TQL: &str = "TQL"; const EVAL: &str = "EVAL"; @@ -128,43 +128,35 @@ impl<'a> ParserContext<'a> { } fn parse_tql_eval(&mut self) -> std::result::Result { - let parser = &mut self.parser; - parser.expect_token(&Token::LParen)?; - let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let delimiter_token = Self::find_next_delimiter_token(parser); - let (step, lookback) = if Self::is_comma(&delimiter_token) { - let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let lookback = Self::parse_string_or_number_or_word(parser, Token::RParen).ok(); - (step, lookback) - } else { - let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; - (step, None) - }; - let query = Self::parse_tql_query(parser, self.sql)?; + let params = self.parse_tql_params()?; + Ok(Statement::Tql(Tql::Eval(TqlEval::from(params)))) + } - Ok(Statement::Tql(Tql::Eval(TqlEval { - start, - end, - step, - lookback, - query, - }))) + fn parse_tql_analyze( + &mut self, + is_verbose: bool, + ) -> std::result::Result { + let mut params = self.parse_tql_params()?; + params.is_verbose = is_verbose; + Ok(Statement::Tql(Tql::Analyze(TqlAnalyze::from(params)))) } fn parse_tql_explain( &mut self, is_verbose: bool, ) -> std::result::Result { - let parser = &mut self.parser; + let mut params = self.parse_tql_params()?; + params.is_verbose = is_verbose; + Ok(Statement::Tql(Tql::Explain(TqlExplain::from(params)))) + } + fn parse_tql_params(&mut self) -> std::result::Result { + let parser = &mut self.parser; let (start, end, step, lookback) = match parser.peek_token().token { Token::LParen => { let _consume_lparen_token = parser.next_token(); - let start = Self::parse_string_or_number_or_word(parser, Token::Comma) - .unwrap_or("0".to_string()); - let end = Self::parse_string_or_number_or_word(parser, Token::Comma) - .unwrap_or("0".to_string()); + let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; + let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; let delimiter_token = Self::find_next_delimiter_token(parser); let (step, lookback) = if Self::is_comma(&delimiter_token) { let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; @@ -178,46 +170,8 @@ impl<'a> ParserContext<'a> { } _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None), }; - let query = Self::parse_tql_query(parser, self.sql)?; - - Ok(Statement::Tql(Tql::Explain(TqlExplain { - query, - start, - end, - step, - lookback, - is_verbose, - }))) - } - - fn parse_tql_analyze( - &mut self, - is_verbose: bool, - ) -> std::result::Result { - let parser = &mut self.parser; - parser.expect_token(&Token::LParen)?; - let start = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let end = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let delimiter_token = Self::find_next_delimiter_token(parser); - let (step, lookback) = if Self::is_comma(&delimiter_token) { - let step = Self::parse_string_or_number_or_word(parser, Token::Comma)?; - let lookback = Self::parse_string_or_number_or_word(parser, Token::RParen).ok(); - (step, lookback) - } else { - let step = Self::parse_string_or_number_or_word(parser, Token::RParen)?; - (step, None) - }; - let query = Self::parse_tql_query(parser, self.sql)?; - - Ok(Statement::Tql(Tql::Analyze(TqlAnalyze { - start, - end, - step, - query, - lookback, - is_verbose, - }))) + Ok(TqlParameters::new(start, end, step, lookback, query)) } fn find_next_delimiter_token(parser: &mut Parser) -> Token { diff --git a/src/sql/src/statements/tql.rs b/src/sql/src/statements/tql.rs index 0261be880bed..f071c19f8e96 100644 --- a/src/sql/src/statements/tql.rs +++ b/src/sql/src/statements/tql.rs @@ -54,3 +54,70 @@ pub struct TqlAnalyze { pub query: String, pub is_verbose: bool, } + +#[derive(Debug)] +pub struct TqlParameters { + start: String, + end: String, + step: String, + lookback: Option, + query: String, + pub is_verbose: bool, +} + +impl TqlParameters { + pub fn new( + start: String, + end: String, + step: String, + lookback: Option, + query: String, + ) -> Self { + TqlParameters { + start, + end, + step, + lookback, + query, + is_verbose: false, + } + } +} + +impl From for TqlEval { + fn from(params: TqlParameters) -> Self { + TqlEval { + start: params.start, + end: params.end, + step: params.step, + lookback: params.lookback, + query: params.query, + } + } +} + +impl From for TqlExplain { + fn from(params: TqlParameters) -> Self { + TqlExplain { + start: params.start, + end: params.end, + step: params.step, + query: params.query, + lookback: params.lookback, + is_verbose: params.is_verbose, + } + } +} + +impl From for TqlAnalyze { + fn from(params: TqlParameters) -> Self { + TqlAnalyze { + start: params.start, + end: params.end, + step: params.step, + query: params.query, + lookback: params.lookback, + is_verbose: params.is_verbose, + } + } +} From 96fc4b92b90ea78bc7ad80685916ffe94c9af2d8 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Tue, 26 Mar 2024 21:13:32 +0000 Subject: [PATCH 14/16] feat(tql): address CR points --- src/sql/src/parsers.rs | 1 + src/sql/src/parsers/error.rs | 56 +++++++++ src/sql/src/parsers/tql_parser.rs | 118 +++++++----------- .../common/tql-explain-analyze/analyze.result | 31 ----- .../common/tql-explain-analyze/analyze.sql | 10 -- .../common/tql-explain-analyze/explain.result | 108 ---------------- .../common/tql-explain-analyze/explain.sql | 8 -- 7 files changed, 104 insertions(+), 228 deletions(-) create mode 100644 src/sql/src/parsers/error.rs diff --git a/src/sql/src/parsers.rs b/src/sql/src/parsers.rs index ca249cf64024..0599914ec98a 100644 --- a/src/sql/src/parsers.rs +++ b/src/sql/src/parsers.rs @@ -18,6 +18,7 @@ pub(crate) mod create_parser; pub(crate) mod delete_parser; pub(crate) mod describe_parser; pub(crate) mod drop_parser; +mod error; pub(crate) mod explain_parser; pub(crate) mod insert_parser; pub(crate) mod query_parser; diff --git a/src/sql/src/parsers/error.rs b/src/sql/src/parsers/error.rs new file mode 100644 index 000000000000..8dcba564c3fc --- /dev/null +++ b/src/sql/src/parsers/error.rs @@ -0,0 +1,56 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion_common::DataFusionError; +use sqlparser::parser::ParserError; + +pub enum TQLError { + Parser(String), + Simplification(String), + Evaluation(String), +} + +impl From for TQLError { + fn from(err: ParserError) -> Self { + TQLError::Parser(err.to_string()) + } +} + +impl From for TQLError { + fn from(err: DataFusionError) -> Self { + match err { + DataFusionError::SQL(parser_err) => TQLError::Parser(parser_err.to_string()), + DataFusionError::Plan(plan_err) => TQLError::Evaluation(plan_err), + unspecified => { + TQLError::Evaluation(format!("Failed to evaluate due to: {unspecified:?}")) + } + } + } +} + +impl From for ParserError { + fn from(tql_err: TQLError) -> Self { + match tql_err { + TQLError::Parser(s) => { + ParserError::ParserError(format!("Failed to parse the query: {s}")) + } + TQLError::Simplification(s) => { + ParserError::ParserError(format!("Failed to simplify the query: {s}")) + } + TQLError::Evaluation(s) => { + ParserError::ParserError(format!("Failed to evaluate the query: {s}")) + } + } + } +} diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 3fcf16de7dc0..69fb4545fae9 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -17,9 +17,7 @@ use std::sync::Arc; use chrono::Utc; use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext}; use datafusion_common::config::ConfigOptions; -use datafusion_common::{ - DFSchema, DataFusionError, Result as DFResult, ScalarValue, TableReference, -}; +use datafusion_common::{DFSchema, Result as DFResult, ScalarValue, TableReference}; use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; use datafusion_physical_expr::execution_props::ExecutionProps; use datafusion_sql::planner::{ContextProvider, SqlToRel}; @@ -42,46 +40,7 @@ use datatypes::arrow::datatypes::DataType; use sqlparser::parser::Parser; use crate::dialect::GreptimeDbDialect; - -pub enum TQLError { - Parser(String), - Simplification(String), - Evaluation(String), -} - -impl From for TQLError { - fn from(err: ParserError) -> Self { - TQLError::Parser(err.to_string()) - } -} - -impl From for TQLError { - fn from(err: DataFusionError) -> Self { - match err { - DataFusionError::SQL(parser_err) => TQLError::Parser(parser_err.to_string()), - DataFusionError::Plan(plan_err) => TQLError::Evaluation(plan_err), - unspecified => { - TQLError::Evaluation(format!("Failed to evaluate due to: {unspecified:?}")) - } - } - } -} - -impl From for ParserError { - fn from(tql_err: TQLError) -> Self { - match tql_err { - TQLError::Parser(s) => { - ParserError::ParserError(format!("Failed to parse the query: {s}")) - } - TQLError::Simplification(s) => { - ParserError::ParserError(format!("Failed to simplify the query: {s}")) - } - TQLError::Evaluation(s) => { - ParserError::ParserError(format!("Failed to evaluate the query: {s}")) - } - } - } -} +use crate::parsers::error::TQLError; /// TQL extension parser, including: /// - `TQL EVAL ` @@ -100,24 +59,34 @@ impl<'a> ParserContext<'a> { if (uppercase == EVAL || uppercase == EVALUATE) && w.quote_style.is_none() => { - self.parse_tql_eval().context(error::SyntaxSnafu) + self.parse_tql_params() + .map(|params| Statement::Tql(Tql::Eval(TqlEval::from(params)))) + .context(error::SyntaxSnafu) } Keyword::EXPLAIN => { - let is_verbose = self.peek_token_as_string() == VERBOSE; + let is_verbose = self.has_verbose_keyword(); if is_verbose { let _consume_verbose_token = self.parser.next_token(); } - self.parse_tql_explain(is_verbose) + self.parse_tql_params() + .map(|mut params| { + params.is_verbose = is_verbose; + Statement::Tql(Tql::Explain(TqlExplain::from(params))) + }) .context(error::SyntaxSnafu) } Keyword::ANALYZE => { - let is_verbose = self.peek_token_as_string() == VERBOSE; + let is_verbose = self.has_verbose_keyword(); if is_verbose { let _consume_verbose_token = self.parser.next_token(); } - self.parse_tql_analyze(is_verbose) + self.parse_tql_params() + .map(|mut params| { + params.is_verbose = is_verbose; + Statement::Tql(Tql::Analyze(TqlAnalyze::from(params))) + }) .context(error::SyntaxSnafu) } _ => self.unsupported(self.peek_token_as_string()), @@ -127,29 +96,6 @@ impl<'a> ParserContext<'a> { } } - fn parse_tql_eval(&mut self) -> std::result::Result { - let params = self.parse_tql_params()?; - Ok(Statement::Tql(Tql::Eval(TqlEval::from(params)))) - } - - fn parse_tql_analyze( - &mut self, - is_verbose: bool, - ) -> std::result::Result { - let mut params = self.parse_tql_params()?; - params.is_verbose = is_verbose; - Ok(Statement::Tql(Tql::Analyze(TqlAnalyze::from(params)))) - } - - fn parse_tql_explain( - &mut self, - is_verbose: bool, - ) -> std::result::Result { - let mut params = self.parse_tql_params()?; - params.is_verbose = is_verbose; - Ok(Statement::Tql(Tql::Explain(TqlExplain::from(params)))) - } - fn parse_tql_params(&mut self) -> std::result::Result { let parser = &mut self.parser; let (start, end, step, lookback) = match parser.peek_token().token { @@ -202,6 +148,10 @@ impl<'a> ParserContext<'a> { matches!(token, Token::RParen) } + fn has_verbose_keyword(&mut self) -> bool { + self.peek_token_as_string().eq_ignore_ascii_case(VERBOSE) + } + fn parse_string_or_number_or_word( parser: &mut Parser, delimiter_token: Token, @@ -562,6 +512,19 @@ mod tests { _ => unreachable!(), } + let sql = "TQL EXPLAIN verbose (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Explain(explain)) => { + assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert_eq!(explain.start, "20"); + assert_eq!(explain.end, "100"); + assert_eq!(explain.step, "10"); + assert_eq!(explain.lookback, None); + assert!(explain.is_verbose); + } + _ => unreachable!(), + } + let sql = "TQL EXPLAIN VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; match parse_into_statement(sql) { Statement::Tql(Tql::Explain(explain)) => { @@ -617,6 +580,19 @@ mod tests { _ => unreachable!(), } + let sql = "TQL ANALYZE verbose (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; + match parse_into_statement(sql) { + Statement::Tql(Tql::Analyze(analyze)) => { + assert_eq!(analyze.start, "1676887657.1"); + assert_eq!(analyze.end, "1676887659.5"); + assert_eq!(analyze.step, "30.3"); + assert_eq!(analyze.lookback, None); + assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"); + assert!(analyze.is_verbose); + } + _ => unreachable!(), + } + let sql = "TQL ANALYZE VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m"; match parse_into_statement(sql) { Statement::Tql(Tql::Analyze(analyze)) => { diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.result b/tests/cases/standalone/common/tql-explain-analyze/analyze.result index a618e1e771b1..96b2548503df 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.result @@ -99,37 +99,6 @@ TQL ANALYZE VERBOSE (0, 10, '5s') test; | REDACTED +-+-+ --- analyze verbose at 0s, 5s and 10s. No point at 0s. --- SQLNESS REPLACE (-+) - --- SQLNESS REPLACE (\s\s+) _ --- SQLNESS REPLACE (elapsed_compute.*) REDACTED --- SQLNESS REPLACE (peers.*) REDACTED --- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED --- SQLNESS REPLACE (metrics.*) REDACTED --- SQLNESS REPLACE (Duration.*) REDACTED -TQL ANALYZE VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; - -+-+-+ -| plan_type_| plan_| -+-+-+ -| Plan with Metrics_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED -|_|_PromSeriesDivideExec: tags=["k"], REDACTED -|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED -|_|_MergeScanExec: REDACTED -|_|_| -| Plan with Full Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED -|_|_PromSeriesDivideExec: tags=["k"], REDACTED -|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED -|_|_MergeScanExec: REDACTED -|_|_| -| Output Rows_| 4_| -| REDACTED -+-+-+ - DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql index 8202bda665a9..e888ba8d51ad 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.sql @@ -38,14 +38,4 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE (Duration.*) REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; --- analyze verbose at 0s, 5s and 10s. No point at 0s. --- SQLNESS REPLACE (-+) - --- SQLNESS REPLACE (\s\s+) _ --- SQLNESS REPLACE (elapsed_compute.*) REDACTED --- SQLNESS REPLACE (peers.*) REDACTED --- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED --- SQLNESS REPLACE (metrics.*) REDACTED --- SQLNESS REPLACE (Duration.*) REDACTED -TQL ANALYZE VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; - DROP TABLE test; diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 8b97c6deb03b..e50d1892f351 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -179,114 +179,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_| +-+-+ --- explain verbose at 0s, 5s and 10s. No point at 0s. --- SQLNESS REPLACE (-+) - --- SQLNESS REPLACE (\s\s+) _ --- SQLNESS REPLACE (elapsed_compute.*) REDACTED --- SQLNESS REPLACE (peers.*) REDACTED --- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -TQL EXPLAIN VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; - -+-+-+ -| plan_type_| plan_| -+-+-+ -| initial_logical_plan_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivide: tags=["k"]_| -|_|_Sort: test.k DESC NULLS LAST, test.j DESC NULLS LAST_| -|_|_Filter: test.j >= TimestampMillisecond(-300000, None) AND test.j <= TimestampMillisecond(300000, None) | -|_|_TableScan: test_| -| logical_plan after count_wildcard_rule_| SAME TEXT AS ABOVE_| -| logical_plan after StringNormalizationRule_| SAME TEXT AS ABOVE_| -| logical_plan after inline_table_scan_| SAME TEXT AS ABOVE_| -| logical_plan after type_coercion_| SAME TEXT AS ABOVE_| -| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivide: tags=["k"]_| -|_|_MergeScan [is_placeholder=false]_| -| analyzed_logical_plan_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_nested_union_| SAME TEXT AS ABOVE_| -| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| -| logical_plan after unwrap_cast_in_comparison_| SAME TEXT AS ABOVE_| -| logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| -| logical_plan after decorrelate_predicate_subquery_| SAME TEXT AS ABOVE_| -| logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| -| logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| -| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| -| logical_plan after merge_projection_| SAME TEXT AS ABOVE_| -| logical_plan after rewrite_disjunctive_predicate_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_cross_join_| SAME TEXT AS ABOVE_| -| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_limit_| SAME TEXT AS ABOVE_| -| logical_plan after propagate_empty_relation_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_one_union_| SAME TEXT AS ABOVE_| -| logical_plan after filter_null_join_keys_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_outer_join_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_filter_| SAME TEXT AS ABOVE_| -| logical_plan after single_distinct_aggregation_to_group_by | SAME TEXT AS ABOVE_| -| logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| -| logical_plan after unwrap_cast_in_comparison_| SAME TEXT AS ABOVE_| -| logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_projection_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_projection_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| -| logical_plan after OrderHintRule_| SAME TEXT AS ABOVE_| -| logical_plan_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivide: tags=["k"]_| -|_|_MergeScan [is_placeholder=false]_| -| initial_physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after OutputRequirements_| OutputRequirementExec_| -|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| -| physical_plan after join_selection_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceDistribution_| OutputRequirementExec_| -|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| OutputRequirementExec_| -|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after coalesce_batches_| SAME TEXT AS ABOVE_| -| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after PipelineChecker_| SAME TEXT AS ABOVE_| -| physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| -| physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@2 ASC NULLS LAST]_| -|_|_MergeScanExec: REDACTED -|_|_| -+-+-+ - DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.sql b/tests/cases/standalone/common/tql-explain-analyze/explain.sql index aa0e25aaa4e8..cf5618496d65 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.sql +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.sql @@ -27,12 +27,4 @@ TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED TQL EXPLAIN VERBOSE (0, 10, '5s') test; --- explain verbose at 0s, 5s and 10s. No point at 0s. --- SQLNESS REPLACE (-+) - --- SQLNESS REPLACE (\s\s+) _ --- SQLNESS REPLACE (elapsed_compute.*) REDACTED --- SQLNESS REPLACE (peers.*) REDACTED --- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -TQL EXPLAIN VERBOSE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; - DROP TABLE test; From 1281597ec4d6b2ec58b6d1d3779176a955676151 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 27 Mar 2024 10:56:20 +0000 Subject: [PATCH 15/16] feat(tql): use snafu for tql errors, add more docs --- src/sql/src/error.rs | 10 +++++ src/sql/src/parsers.rs | 2 +- src/sql/src/parsers/error.rs | 60 ++++++++++++---------------- src/sql/src/parsers/tql_parser.rs | 66 +++++++++++++++++++------------ src/sql/src/statements/tql.rs | 3 ++ 5 files changed, 81 insertions(+), 60 deletions(-) diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs index b85c164a1d99..208dd6915f68 100644 --- a/src/sql/src/error.rs +++ b/src/sql/src/error.rs @@ -24,6 +24,7 @@ use snafu::{Location, Snafu}; use sqlparser::parser::ParserError; use crate::ast::{Expr, Value as SqlValue}; +use crate::parsers::error::TQLError; pub type Result = std::result::Result; @@ -66,6 +67,14 @@ pub enum Error { location: Location, }, + // Syntax error from tql parser. + #[snafu(display(""))] + TQLSyntax { + #[snafu(source)] + error: TQLError, + location: Location, + }, + #[snafu(display("Missing time index constraint"))] MissingTimeIndex {}, @@ -170,6 +179,7 @@ impl ErrorExt for Error { UnsupportedDefaultValue { .. } | Unsupported { .. } => StatusCode::Unsupported, Unexpected { .. } | Syntax { .. } + | TQLSyntax { .. } | MissingTimeIndex { .. } | InvalidTimeIndex { .. } | InvalidSql { .. } diff --git a/src/sql/src/parsers.rs b/src/sql/src/parsers.rs index 0599914ec98a..b7e5c8c44e84 100644 --- a/src/sql/src/parsers.rs +++ b/src/sql/src/parsers.rs @@ -18,7 +18,7 @@ pub(crate) mod create_parser; pub(crate) mod delete_parser; pub(crate) mod describe_parser; pub(crate) mod drop_parser; -mod error; +pub(crate) mod error; pub(crate) mod explain_parser; pub(crate) mod insert_parser; pub(crate) mod query_parser; diff --git a/src/sql/src/parsers/error.rs b/src/sql/src/parsers/error.rs index 8dcba564c3fc..bdb469ac2bae 100644 --- a/src/sql/src/parsers/error.rs +++ b/src/sql/src/parsers/error.rs @@ -12,45 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. +use common_macro::stack_trace_debug; use datafusion_common::DataFusionError; +use snafu::{Location, Snafu}; use sqlparser::parser::ParserError; +/// TQL parser & evaluation errors. +#[derive(Snafu)] +#[snafu(visibility(pub))] +#[stack_trace_debug] pub enum TQLError { - Parser(String), - Simplification(String), - Evaluation(String), -} + #[snafu(display("Failed to parse TQL expression"))] + Parser { + #[snafu(source)] + error: ParserError, + location: Location, + }, -impl From for TQLError { - fn from(err: ParserError) -> Self { - TQLError::Parser(err.to_string()) - } -} + #[snafu(display("Failed to convert to logical TQL expression"))] + ConvertToLogicalExpression { + #[snafu(source)] + error: DataFusionError, + location: Location, + }, -impl From for TQLError { - fn from(err: DataFusionError) -> Self { - match err { - DataFusionError::SQL(parser_err) => TQLError::Parser(parser_err.to_string()), - DataFusionError::Plan(plan_err) => TQLError::Evaluation(plan_err), - unspecified => { - TQLError::Evaluation(format!("Failed to evaluate due to: {unspecified:?}")) - } - } - } -} + #[snafu(display("Failed to simplify TQL expression"))] + Simplification { + #[snafu(source)] + error: DataFusionError, + location: Location, + }, -impl From for ParserError { - fn from(tql_err: TQLError) -> Self { - match tql_err { - TQLError::Parser(s) => { - ParserError::ParserError(format!("Failed to parse the query: {s}")) - } - TQLError::Simplification(s) => { - ParserError::ParserError(format!("Failed to simplify the query: {s}")) - } - TQLError::Evaluation(s) => { - ParserError::ParserError(format!("Failed to evaluate the query: {s}")) - } - } - } + #[snafu(display("Failed to evaluate TQL expression: {}", msg))] + Evaluation { msg: String }, } diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 69fb4545fae9..6e21d0613304 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -21,7 +21,7 @@ use datafusion_common::{DFSchema, Result as DFResult, ScalarValue, TableReferenc use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; use datafusion_physical_expr::execution_props::ExecutionProps; use datafusion_sql::planner::{ContextProvider, SqlToRel}; -use snafu::ResultExt; +use snafu::{OptionExt, ResultExt}; use sqlparser::keywords::Keyword; use sqlparser::parser::ParserError; use sqlparser::tokenizer::Token; @@ -40,7 +40,9 @@ use datatypes::arrow::datatypes::DataType; use sqlparser::parser::Parser; use crate::dialect::GreptimeDbDialect; -use crate::parsers::error::TQLError; +use crate::parsers::error::{ + ConvertToLogicalExpressionSnafu, EvaluationSnafu, ParserSnafu, SimplificationSnafu, TQLError, +}; /// TQL extension parser, including: /// - `TQL EVAL ` @@ -61,7 +63,7 @@ impl<'a> ParserContext<'a> { { self.parse_tql_params() .map(|params| Statement::Tql(Tql::Eval(TqlEval::from(params)))) - .context(error::SyntaxSnafu) + .context(error::TQLSyntaxSnafu) } Keyword::EXPLAIN => { @@ -74,7 +76,7 @@ impl<'a> ParserContext<'a> { params.is_verbose = is_verbose; Statement::Tql(Tql::Explain(TqlExplain::from(params))) }) - .context(error::SyntaxSnafu) + .context(error::TQLSyntaxSnafu) } Keyword::ANALYZE => { @@ -87,7 +89,7 @@ impl<'a> ParserContext<'a> { params.is_verbose = is_verbose; Statement::Tql(Tql::Analyze(TqlAnalyze::from(params))) }) - .context(error::SyntaxSnafu) + .context(error::TQLSyntaxSnafu) } _ => self.unsupported(self.peek_token_as_string()), } @@ -96,7 +98,7 @@ impl<'a> ParserContext<'a> { } } - fn parse_tql_params(&mut self) -> std::result::Result { + fn parse_tql_params(&mut self) -> std::result::Result { let parser = &mut self.parser; let (start, end, step, lookback) = match parser.peek_token().token { Token::LParen => { @@ -116,7 +118,7 @@ impl<'a> ParserContext<'a> { } _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None), }; - let query = Self::parse_tql_query(parser, self.sql)?; + let query = Self::parse_tql_query(parser, self.sql).context(ParserSnafu {})?; Ok(TqlParameters::new(start, end, step, lookback, query)) } @@ -163,23 +165,29 @@ impl<'a> ParserContext<'a> { tokens.push(token.token); } let result = match tokens.len() { - 0 => Err(TQLError::Parser("Expected tokens".to_string())), + 0 => Err(ParserError::ParserError( + "Expected at least one token".to_string(), + )) + .context(ParserSnafu {}), 1 => { let value = match tokens[0].clone() { Token::Number(n, _) => n, Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s, Token::Word(_) => Self::parse_tokens(tokens)?, unexpected => { - return Err(TQLError::Parser(format!( - "Expect number, string or word, but is {unexpected:?}" - ))); + return Err(ParserError::ParserError(format!( + "Expected number, string or word, but have {unexpected:?}" + ))) + .context(ParserSnafu {}); } }; Ok(value) } _ => Self::parse_tokens(tokens), }; - parser.expect_token(&delimiter_token)?; + parser + .expect_token(&delimiter_token) + .context(ParserSnafu {})?; result } @@ -194,16 +202,14 @@ impl<'a> ParserContext<'a> { Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) .parse_expr() - .map_err(|err| TQLError::Parser(format!("Failed to convert to expression: {err:?}"))) + .context(ParserSnafu {}) } fn parse_to_logical_expr(expr: sqlparser::ast::Expr) -> std::result::Result { let empty_df_schema = DFSchema::empty(); SqlToRel::new(&StubContextProvider {}) .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) - .map_err(|err| { - TQLError::Parser(format!("Failed to convert to logical expression {err:?}")) - }) + .context(ConvertToLogicalExpressionSnafu {}) } fn simplify_expr(logical_expr: Expr) -> std::result::Result { @@ -212,21 +218,31 @@ impl<'a> ParserContext<'a> { let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); ExprSimplifier::new(info) .simplify(logical_expr) - .map_err(|err| { - TQLError::Simplification(format!("Failed to simplify expression {err:?}")) - }) + .context(SimplificationSnafu {}) } fn evaluate_expr(simplified_expr: Expr) -> std::result::Result { match simplified_expr { - Expr::Literal(ScalarValue::TimestampNanosecond(v, _)) - | Expr::Literal(ScalarValue::DurationNanosecond(v)) => v, + Expr::Literal(ScalarValue::TimestampNanosecond(ts_nanos, _)) + | Expr::Literal(ScalarValue::DurationNanosecond(ts_nanos)) => { + ts_nanos.map(|v| v / 1_000_000_000) + } + Expr::Literal(ScalarValue::TimestampMicrosecond(ts_micros, _)) + | Expr::Literal(ScalarValue::DurationMicrosecond(ts_micros)) => { + ts_micros.map(|v| v / 1_000_000) + } + Expr::Literal(ScalarValue::TimestampMillisecond(ts_millis, _)) + | Expr::Literal(ScalarValue::DurationMillisecond(ts_millis)) => { + ts_millis.map(|v| v / 1_000) + } + Expr::Literal(ScalarValue::TimestampSecond(ts_secs, _)) + | Expr::Literal(ScalarValue::DurationSecond(ts_secs)) => ts_secs, _ => None, } - .map(|timestamp_nanos| (timestamp_nanos / 1_000_000_000).to_string()) - .ok_or(TQLError::Evaluation(format!( - "Failed to extract a timestamp value {simplified_expr:?}" - ))) + .map(|ts| ts.to_string()) + .context(EvaluationSnafu { + msg: format!("Failed to extract a timestamp value {simplified_expr:?}"), + }) } fn parse_tql_query(parser: &mut Parser, sql: &str) -> std::result::Result { diff --git a/src/sql/src/statements/tql.rs b/src/sql/src/statements/tql.rs index f071c19f8e96..6bc4136068ea 100644 --- a/src/sql/src/statements/tql.rs +++ b/src/sql/src/statements/tql.rs @@ -55,6 +55,9 @@ pub struct TqlAnalyze { pub is_verbose: bool, } +/// Intermediate structure used to unify parameter mappings for various TQL operations. +/// This struct serves as a common parameter container for parsing TQL queries +/// and constructing corresponding TQL operations: `TqlEval`, `TqlAnalyze` or `TqlExplain`. #[derive(Debug)] pub struct TqlParameters { start: String, From 368276b11a946fe7598eda65850c261dcbdec39c Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 27 Mar 2024 13:08:40 +0000 Subject: [PATCH 16/16] feat(tql): address CR points --- src/sql/src/parsers/tql_parser.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index 6e21d0613304..a681ca10012c 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -118,7 +118,7 @@ impl<'a> ParserContext<'a> { } _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None), }; - let query = Self::parse_tql_query(parser, self.sql).context(ParserSnafu {})?; + let query = Self::parse_tql_query(parser, self.sql).context(ParserSnafu)?; Ok(TqlParameters::new(start, end, step, lookback, query)) } @@ -168,7 +168,7 @@ impl<'a> ParserContext<'a> { 0 => Err(ParserError::ParserError( "Expected at least one token".to_string(), )) - .context(ParserSnafu {}), + .context(ParserSnafu), 1 => { let value = match tokens[0].clone() { Token::Number(n, _) => n, @@ -178,16 +178,14 @@ impl<'a> ParserContext<'a> { return Err(ParserError::ParserError(format!( "Expected number, string or word, but have {unexpected:?}" ))) - .context(ParserSnafu {}); + .context(ParserSnafu); } }; Ok(value) } _ => Self::parse_tokens(tokens), }; - parser - .expect_token(&delimiter_token) - .context(ParserSnafu {})?; + parser.expect_token(&delimiter_token).context(ParserSnafu)?; result } @@ -202,14 +200,14 @@ impl<'a> ParserContext<'a> { Parser::new(&GreptimeDbDialect {}) .with_tokens(tokens) .parse_expr() - .context(ParserSnafu {}) + .context(ParserSnafu) } fn parse_to_logical_expr(expr: sqlparser::ast::Expr) -> std::result::Result { let empty_df_schema = DFSchema::empty(); SqlToRel::new(&StubContextProvider {}) .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) - .context(ConvertToLogicalExpressionSnafu {}) + .context(ConvertToLogicalExpressionSnafu) } fn simplify_expr(logical_expr: Expr) -> std::result::Result { @@ -218,7 +216,7 @@ impl<'a> ParserContext<'a> { let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema)); ExprSimplifier::new(info) .simplify(logical_expr) - .context(SimplificationSnafu {}) + .context(SimplificationSnafu) } fn evaluate_expr(simplified_expr: Expr) -> std::result::Result {