feat(parser): the 1st version of a new combinator style parser

src/sqlparser/Cargo.toml

@@ -29,6 +29,7 @@ itertools = { workspace = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 tracing = "0.1"
 tracing-subscriber = "0.3"
+winnow = { version = "0.6.8", git = "https://github.com/TennyZhuang/winnow.git", rev = "509121449b451f0faa1b698b4e8cb36dfaa1a4b5" }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }

src/sqlparser/src/lib.rs

@@ -45,6 +45,7 @@ extern crate alloc;
 pub mod ast;
 pub mod keywords;
 pub mod parser;
+pub mod parser_v2;
 pub mod tokenizer;
 
 #[doc(hidden)]

src/sqlparser/src/parser.rs

@@ -21,6 +21,7 @@ use alloc::{
     vec::Vec,
 };
 use core::fmt;
+use std::fmt::format;
 
 use itertools::Itertools;
 use tracing::{debug, instrument};
@@ -187,6 +188,29 @@ impl Parser {
         }
     }
 
+    pub(crate) fn parse_v2<'a, O>(
+        &'a mut self,
+        mut parse_next: impl winnow::Parser<
+            winnow::Located<&'a [TokenWithLocation]>,
+            O,
+            winnow::error::ContextError,
+        >,
+    ) -> Result<O, ParserError> {
+        use winnow::stream::Location;
+
+        let mut token_stream = winnow::Located::new(&*self.tokens);
+        let output = parse_next.parse_next(&mut token_stream).map_err(|e| {
+            ParserError::ParserError(format!(
+                "Error parsing SQL at {}: {}",
+                token_stream.location(),
+                e
+            ))
+        });
+        let offset = token_stream.location();
+        self.index += offset;
+        output
+    }
+
     /// Parse a SQL statement and produce an Abstract Syntax Tree (AST)
     #[instrument(level = "debug")]
     pub fn parse_sql(sql: &str) -> Result<Vec<Statement>, ParserError> {

src/sqlparser/src/parser_v2/data_type.rs

@@ -0,0 +1,161 @@
+use core::cell::RefCell;
+use std::rc::Rc;
+
+use winnow::combinator::{alt, delimited, dispatch, empty, fail, opt, separated, seq};
+use winnow::error::{ContextError, ErrMode};
+use winnow::{PResult, Parser, Stateful};
+
+use super::{
+    identifier_non_reserved, keyword, literal_uint, precision_in_range, token, with_state,
+    TokenStream,
+};
+use crate::ast::{DataType, StructField};
+use crate::keywords::Keyword;
+use crate::tokenizer::Token;
+
+#[derive(Default, Debug)]
+struct DataTypeParsingState {
+    /// When we consumed an [`Token::ShiftRight`], we set this to true.
+    remaining_close: Rc<RefCell<bool>>,
+}
+
+type StatefulStream<S> = Stateful<S, DataTypeParsingState>;
+
+fn struct_data_type<S>(input: &mut StatefulStream<S>) -> PResult<Vec<StructField>>
+where
+    S: TokenStream,
+{
+    let remaining_close1 = input.state.remaining_close.clone();
+    let remaining_close2 = input.state.remaining_close.clone();
+
+    let consume_close = alt((
+        move |_input: &mut StatefulStream<S>| -> PResult<()> {
+            if *remaining_close1.borrow() {
+                Ok(())
+            } else {
+                Err(ErrMode::Backtrack(ContextError::new()))
+            }
+        }
+        .void(),
+        (
+            Token::ShiftRight,
+            move |_input: &mut StatefulStream<S>| -> PResult<()> {
+                *remaining_close2.borrow_mut() = true;
+                Ok(())
+            },
+        )
+            .void(),
+        Token::Gt.void(),
+    ));
+
+    delimited(
+        Token::Lt,
+        separated(
+            1..,
+            seq! {
+                StructField {
+                    name: identifier_non_reserved,
+                    _: Token::Colon,
+                    data_type: data_type_stateful,
+                }
+            },
+            Token::Comma,
+        ),
+        consume_close,
+    )
+    .parse_next(input)
+}
+
+pub fn data_type<S>(input: &mut S) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    with_state::<S, DataTypeParsingState, _, _>(data_type_stateful).parse_next(input)
+}
+
+fn data_type_stateful<S>(input: &mut StatefulStream<S>) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    (
+        data_type_stateful_inner,
+        opt((Token::LBracket, Token::RBracket)),
+    )
+        .map(|(dt, is_array)| {
+            if is_array.is_some() {
+                DataType::Array(Box::new(dt))
+            } else {
+                dt
+            }
+        })
+        .parse_next(input)
+}
+
+fn data_type_stateful_inner<S>(input: &mut StatefulStream<S>) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    let with_time_zone = || {
+        opt(alt((
+            (Keyword::WITH, Keyword::TIME, Keyword::ZONE).value(true),
+            (Keyword::WITHOUT, Keyword::TIME, Keyword::ZONE).value(false),
+        )))
+        .map(|x| x.unwrap_or(false))
+    };
+
+    let precision_and_scale = || {
+        opt(delimited(
+            Token::LParen,
+            (
+                literal_uint,
+                opt((Token::Comma, literal_uint).map(|(_, x)| x)),
+            ),
+            Token::RParen,
+        ))
+        .map(|p| match p {
+            Some((x, y)) => (Some(x), y),
+            None => (None, None),
+        })
+    };
+
+    let keywords = dispatch! {keyword;
+        Keyword::BOOLEAN | Keyword::BOOL => empty.value(DataType::Boolean),
+        Keyword::FLOAT => opt(precision_in_range(1..53)).map(|precision| DataType::Float(precision)),
+        Keyword::REAL => empty.value(DataType::Real),
+        Keyword::DOUBLE => Keyword::PRECISION.value(DataType::Double),
+        Keyword::SMALLINT => empty.value(DataType::SmallInt),
+        Keyword::INT | Keyword::INTEGER => empty.value(DataType::Int),
+        Keyword::BIGINT => empty.value(DataType::BigInt),
+        Keyword::STRING | Keyword::VARCHAR => empty.value(DataType::Varchar),
+        Keyword::CHAR | Keyword::CHARACTER => dispatch! {keyword;
+            Keyword::VARYING => empty.value(DataType::Varchar),
+            _ => opt(precision_in_range(..)).map(|precision| DataType::Char(precision)),
+        },
+        Keyword::UUID => empty.value(DataType::Uuid),
+        Keyword::DATE => empty.value(DataType::Date),
+        Keyword::TIMESTAMP => with_time_zone().map(|with_tz| DataType::Timestamp(with_tz)),
+        Keyword::TIME => with_time_zone().map(|with_tz| DataType::Time(with_tz)),
+        // TODO: Support complex inverval type parsing.
+        Keyword::INTERVAL => empty.value(DataType::Interval),
+        Keyword::REGCLASS => empty.value(DataType::Regclass),
+        Keyword::REGPROC => empty.value(DataType::Regproc),
+        Keyword::STRUCT => struct_data_type.map(DataType::Struct),
+        Keyword::BYTEA => empty.value(DataType::Bytea),
+        Keyword::NUMERIC | Keyword::DECIMAL | Keyword::DEC => precision_and_scale().map(|(precision, scale)| {
+            DataType::Decimal(precision, scale)
+        }),
+        _ => fail::<_, DataType, _>,
+    };
+
+    alt((
+        keywords,
+        // JSONB is not a keyword, but a special data type.
+        token
+            .verify(|t| match &t.token {
+                Token::Word(w) if w.value.eq_ignore_ascii_case("jsonb") => true,
+                _ => false,
+            })
+            .value(DataType::Jsonb),
+    ))
+    .parse_next(input)
+}

src/sqlparser/src/parser_v2/mod.rs

@@ -0,0 +1,142 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use winnow::error::ContextError;
+use winnow::stream::{Location, Stream, StreamIsPartial};
+use winnow::token::any;
+use winnow::{PResult, Parser, Stateful};
+
+use crate::ast::Ident;
+use crate::keywords::{self, Keyword};
+use crate::tokenizer::{Token, TokenWithLocation};
+
+mod data_type;
+mod number;
+
+pub(crate) use data_type::*;
+pub(crate) use number::*;
+
+trait TokenStream: Stream<Token = TokenWithLocation> + StreamIsPartial + Location + Default {}
+
+impl<S> TokenStream for S where
+    S: Stream<Token = TokenWithLocation> + StreamIsPartial + Location + Default
+{
+}
+
+fn token<S>(input: &mut S) -> PResult<TokenWithLocation>
+where
+    S: TokenStream,
+{
+    any(input)
+}
+
+fn keyword<S>(input: &mut S) -> PResult<Keyword>
+where
+    S: TokenStream,
+{
+    token
+        .verify_map(|t| match &t.token {
+            Token::Word(w) if w.keyword != Keyword::NoKeyword => Some(w.keyword),
+            _ => None,
+        })
+        .parse_next(input)
+}
+
+impl<I> Parser<I, TokenWithLocation, ContextError> for Token
+where
+    I: TokenStream,
+{
+    fn parse_next(&mut self, input: &mut I) -> PResult<TokenWithLocation, ContextError> {
+        token
+            .verify(move |t: &TokenWithLocation| t.token == *self)
+            .parse_next(input)
+    }
+}
+
+impl<I> Parser<I, Keyword, ContextError> for Keyword
+where
+    I: TokenStream,
+{
+    fn parse_next(&mut self, input: &mut I) -> PResult<Keyword, ContextError> {
+        token
+            .verify_map(move |t| match &t.token {
+                Token::Word(w) if *self == w.keyword => Some(w.keyword),
+                _ => None,
+            })
+            .parse_next(input)
+    }
+}
+
+fn identifier_non_reserved<S>(input: &mut S) -> PResult<Ident>
+where
+    S: TokenStream,
+{
+    // FIXME: Reporting error correctly.
+    token
+        .verify_map(|t| match &t.token {
+            Token::Word(w) if !keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&w.keyword) => {
+                w.to_ident().ok()
+            }
+            _ => None,
+        })
+        .parse_next(input)
+}
+
+fn with_state<S, State, O, ParseNext>(mut parse_next: ParseNext) -> impl Parser<S, O, ContextError>
+where
+    S: TokenStream,
+    State: Default,
+    ParseNext: Parser<Stateful<S, State>, O, ContextError>,
+{
+    move |input: &mut S| -> PResult<O> {
+        let state = State::default();
+        let input2 = std::mem::take(input);
+        let mut stateful = Stateful {
+            input: input2,
+            state,
+        };
+        let output = parse_next.parse_next(&mut stateful);
+        *input = stateful.input;
+        output
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use winnow::Located;
+
+    use super::*;
+    use crate::tokenizer::Tokenizer;
+
+    #[test]
+    fn test_basic() {
+        let input = "SELECT 1";
+        let tokens = Tokenizer::new(input).tokenize_with_location().unwrap();
+        let mut token_stream = Located::new(&*tokens);
+        Token::make_keyword("SELECT")
+            .parse_next(&mut token_stream)
+            .unwrap();
+    }
+
+    #[test]
+    fn test_stateful() {
+        let input = "SELECT 1";
+        let tokens = Tokenizer::new(input).tokenize_with_location().unwrap();
+        let mut token_stream = Located::new(&*tokens);
+        with_state(|input: &mut Stateful<_, usize>| -> PResult<()> {
+            input.state += 1;
+            Token::make_keyword("SELECT").void().parse_next(input)
+        })
+        .parse_next(&mut token_stream)
+        .unwrap();
+    }
+}