From 36197461bf09e2aa80c78fcd9e37a5d166962152 Mon Sep 17 00:00:00 2001 From: Igor Matuszewski Date: Mon, 4 Dec 2023 12:06:15 +0100 Subject: [PATCH] Don't explicitly specify not_followed_by to disambiguate scanners (#688) Part of #638 This leverages the existing trie construction for the literal scanners. By not explicitly disambiguating ourselves, we won't have to be careful about missing something when updating the grammar. cc @OmarTawfik since we talked about this recently --- .../inputs/language/src/definition.rs | 112 ++------ .../cargo/crate/src/generated/language.rs | 263 ++++++------------ .../src/cst_output/generated/Statements.rs | 5 + .../npm/crate/src/generated/language.rs | 263 ++++++------------ .../generated/0.4.11-success.yml | 138 +++++++++ .../Statements/compound_tokens/input.sol | 13 + 6 files changed, 348 insertions(+), 446 deletions(-) create mode 100644 crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/generated/0.4.11-success.yml create mode 100644 crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/input.sol diff --git a/crates/solidity/inputs/language/src/definition.rs b/crates/solidity/inputs/language/src/definition.rs index f1676d051b..e23c90b648 100644 --- a/crates/solidity/inputs/language/src/definition.rs +++ b/crates/solidity/inputs/language/src/definition.rs @@ -1825,12 +1825,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Colon, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom(":"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom(":"))] ), Token( name = ColonEqual, @@ -1838,12 +1833,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Equal, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("="), - not_followed_by = Choice([Atom("="), Atom(">")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("="))] ), Token( name = EqualEqual, @@ -1855,12 +1845,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Asterisk, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("*"), - not_followed_by = Choice([Atom("="), Atom("*")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("*"))] ), Token( name = AsteriskEqual, @@ -1872,12 +1857,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Bar, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("|"), - not_followed_by = Choice([Atom("="), Atom("|")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("|"))] ), Token( name = BarEqual, @@ -1889,12 +1869,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Ampersand, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("&"), - not_followed_by = Choice([Atom("="), Atom("&")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("&"))] ), Token( name = AmpersandEqual, @@ -1906,12 +1881,7 @@ codegen_language_macros::compile!(Language( ), Token( name = LessThan, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("<"), - not_followed_by = Choice([Atom("="), Atom("<")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("<"))] ), Token( name = LessThanEqual, @@ -1919,12 +1889,7 @@ codegen_language_macros::compile!(Language( ), Token( name = LessThanLessThan, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("<<"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom("<<"))] ), Token( name = LessThanLessThanEqual, @@ -1932,12 +1897,7 @@ codegen_language_macros::compile!(Language( ), Token( name = GreaterThan, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom(">"), - not_followed_by = Choice([Atom("="), Atom(">")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom(">"))] ), Token( name = GreaterThanEqual, @@ -1945,12 +1905,7 @@ codegen_language_macros::compile!(Language( ), Token( name = GreaterThanGreaterThan, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom(">>"), - not_followed_by = Choice([Atom("="), Atom(">")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom(">>"))] ), Token( name = GreaterThanGreaterThanEqual, @@ -1958,12 +1913,7 @@ codegen_language_macros::compile!(Language( ), Token( name = GreaterThanGreaterThanGreaterThan, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom(">>>"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom(">>>"))] ), Token( name = GreaterThanGreaterThanGreaterThanEqual, @@ -1971,12 +1921,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Plus, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("+"), - not_followed_by = Choice([Atom("="), Atom("+")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("+"))] ), Token( name = PlusEqual, @@ -1988,12 +1933,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Minus, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("-"), - not_followed_by = Choice([Atom("="), Atom("-"), Atom(">")]) - ) - )] + definitions = [TokenDefinition(scanner = Atom("-"))] ), Token( name = MinusEqual, @@ -2009,12 +1949,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Slash, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("/"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom("/"))] ), Token( name = SlashEqual, @@ -2022,12 +1957,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Percent, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("%"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom("%"))] ), Token( name = PercentEqual, @@ -2035,12 +1965,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Bang, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("!"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom("!"))] ), Token( name = BangEqual, @@ -2048,12 +1973,7 @@ codegen_language_macros::compile!(Language( ), Token( name = Caret, - definitions = [TokenDefinition( - scanner = TrailingContext( - scanner = Atom("^"), - not_followed_by = Atom("=") - ) - )] + definitions = [TokenDefinition(scanner = Atom("^"))] ), Token( name = CaretEqual, diff --git a/crates/solidity/outputs/cargo/crate/src/generated/language.rs b/crates/solidity/outputs/cargo/crate/src/generated/language.rs index d25802227e..176234241c 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/language.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/language.rs @@ -5594,15 +5594,6 @@ impl Language { * Scanner Functions ********************************************/ - #[allow(unused_assignments, unused_parens)] - fn ampersand(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '&'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '&')) - ) - } - #[allow(unused_assignments, unused_parens)] fn ascii_escape(&self, input: &mut ParserContext) -> bool { scan_choice!( @@ -5627,29 +5618,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn asterisk(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '*'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '*')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn bang(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '!'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn bar(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '|'), - scan_choice!(input, scan_chars!(input, '|'), scan_chars!(input, '=')) - ) - } - #[allow(unused_assignments, unused_parens)] fn bytes_keyword(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -5695,16 +5663,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn caret(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '^'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn colon(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, ':'), scan_chars!(input, '=')) - } - #[allow(unused_assignments, unused_parens)] fn decimal_digits(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -5837,15 +5795,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn equal(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '='), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - #[allow(unused_assignments, unused_parens)] fn escape_sequence(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6142,33 +6091,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>'), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn greater_than_greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>', '>'), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn greater_than_greater_than_greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>', '>', '>'), - scan_chars!(input, '=') - ) - } - #[allow(unused_assignments, unused_parens)] fn hex_byte_escape(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6324,34 +6246,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn less_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '<'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '<')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn less_than_less_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '<', '<'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn minus(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '-'), - scan_choice!( - input, - scan_chars!(input, '>'), - scan_chars!(input, '='), - scan_chars!(input, '-') - ) - ) - } - #[allow(unused_assignments, unused_parens)] fn multiline_comment(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6370,20 +6264,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn percent(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '%'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn plus(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '+'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '+')) - ) - } - #[allow(unused_assignments, unused_parens)] fn raw_identifier(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6447,11 +6327,6 @@ impl Language { } } - #[allow(unused_assignments, unused_parens)] - fn slash(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '/'), scan_chars!(input, '=')) - } - #[allow(unused_assignments, unused_parens)] fn ufixed_keyword(&self, input: &mut ParserContext) -> bool { scan_choice!( @@ -8644,16 +8519,30 @@ impl Lexer for Language { input.set_position(save); if let Some(kind) = match input.next() { - Some('!') => scan_chars!(input, '=').then_some(TokenKind::BangEqual), - Some('%') => scan_chars!(input, '=').then_some(TokenKind::PercentEqual), + Some('!') => match input.next() { + Some('=') => Some(TokenKind::BangEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Bang) + } + None => Some(TokenKind::Bang), + }, + Some('%') => match input.next() { + Some('=') => Some(TokenKind::PercentEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Percent) + } + None => Some(TokenKind::Percent), + }, Some('&') => match input.next() { Some('&') => Some(TokenKind::AmpersandAmpersand), Some('=') => Some(TokenKind::AmpersandEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Ampersand) } - None => None, + None => Some(TokenKind::Ampersand), }, Some('(') => Some(TokenKind::OpenParen), Some(')') => Some(TokenKind::CloseParen), @@ -8662,18 +8551,18 @@ impl Lexer for Language { Some('=') => Some(TokenKind::AsteriskEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Asterisk) } - None => None, + None => Some(TokenKind::Asterisk), }, Some('+') => match input.next() { Some('+') => Some(TokenKind::PlusPlus), Some('=') => Some(TokenKind::PlusEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Plus) } - None => None, + None => Some(TokenKind::Plus), }, Some(',') => Some(TokenKind::Comma), Some('-') => match input.next() { @@ -8681,64 +8570,92 @@ impl Lexer for Language { Some('=') => Some(TokenKind::MinusEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Minus) } - None => None, + None => Some(TokenKind::Minus), }, Some('.') => Some(TokenKind::Period), - Some('/') => scan_chars!(input, '=').then_some(TokenKind::SlashEqual), + Some('/') => match input.next() { + Some('=') => Some(TokenKind::SlashEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Slash) + } + None => Some(TokenKind::Slash), + }, + Some(':') => Some(TokenKind::Colon), Some(';') => Some(TokenKind::Semicolon), Some('<') => match input.next() { - Some('<') => { - scan_chars!(input, '=').then_some(TokenKind::LessThanLessThanEqual) - } + Some('<') => match input.next() { + Some('=') => Some(TokenKind::LessThanLessThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::LessThanLessThan) + } + None => Some(TokenKind::LessThanLessThan), + }, Some('=') => Some(TokenKind::LessThanEqual), Some(_) => { input.undo(); - None + Some(TokenKind::LessThan) } - None => None, + None => Some(TokenKind::LessThan), }, Some('=') => match input.next() { Some('=') => Some(TokenKind::EqualEqual), Some('>') => Some(TokenKind::EqualGreaterThan), Some(_) => { input.undo(); - None + Some(TokenKind::Equal) } - None => None, + None => Some(TokenKind::Equal), }, Some('>') => match input.next() { Some('=') => Some(TokenKind::GreaterThanEqual), Some('>') => match input.next() { Some('=') => Some(TokenKind::GreaterThanGreaterThanEqual), - Some('>') => scan_chars!(input, '=') - .then_some(TokenKind::GreaterThanGreaterThanGreaterThanEqual), + Some('>') => match input.next() { + Some('=') => { + Some(TokenKind::GreaterThanGreaterThanGreaterThanEqual) + } + Some(_) => { + input.undo(); + Some(TokenKind::GreaterThanGreaterThanGreaterThan) + } + None => Some(TokenKind::GreaterThanGreaterThanGreaterThan), + }, Some(_) => { input.undo(); - None + Some(TokenKind::GreaterThanGreaterThan) } - None => None, + None => Some(TokenKind::GreaterThanGreaterThan), }, Some(_) => { input.undo(); - None + Some(TokenKind::GreaterThan) } - None => None, + None => Some(TokenKind::GreaterThan), }, Some('?') => Some(TokenKind::QuestionMark), Some('[') => Some(TokenKind::OpenBracket), Some(']') => Some(TokenKind::CloseBracket), - Some('^') => scan_chars!(input, '=').then_some(TokenKind::CaretEqual), + Some('^') => match input.next() { + Some('=') => Some(TokenKind::CaretEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Caret) + } + None => Some(TokenKind::Caret), + }, Some('{') => Some(TokenKind::OpenBrace), Some('|') => match input.next() { Some('=') => Some(TokenKind::BarEqual), Some('|') => Some(TokenKind::BarBar), Some(_) => { input.undo(); - None + Some(TokenKind::Bar) } - None => None, + None => Some(TokenKind::Bar), }, Some('}') => Some(TokenKind::CloseBrace), Some('~') => Some(TokenKind::Tilde), @@ -8754,32 +8671,16 @@ impl Lexer for Language { input.set_position(save); longest_match! { - { Ampersand = ampersand } { AsciiStringLiteral = ascii_string_literal } - { Asterisk = asterisk } - { Bang = bang } - { Bar = bar } { BytesKeyword = bytes_keyword } - { Caret = caret } - { Colon = colon } { DecimalLiteral = decimal_literal } { EndOfLine = end_of_line } - { Equal = equal } { FixedKeyword = fixed_keyword } - { GreaterThan = greater_than } - { GreaterThanGreaterThan = greater_than_greater_than } - { GreaterThanGreaterThanGreaterThan = greater_than_greater_than_greater_than } { HexLiteral = hex_literal } { HexStringLiteral = hex_string_literal } { IntKeyword = int_keyword } - { LessThan = less_than } - { LessThanLessThan = less_than_less_than } - { Minus = minus } { MultilineComment = multiline_comment } - { Percent = percent } - { Plus = plus } { SingleLineComment = single_line_comment } - { Slash = slash } { UfixedKeyword = ufixed_keyword } { UintKeyword = uint_keyword } { UnicodeStringLiteral = unicode_string_literal } @@ -8826,10 +8727,27 @@ impl Lexer for Language { input.set_position(save); if let Some(kind) = match input.next() { + Some('-') => Some(TokenKind::Minus), Some('.') => Some(TokenKind::Period), Some(';') => Some(TokenKind::Semicolon), - Some('<') => scan_chars!(input, '=').then_some(TokenKind::LessThanEqual), - Some('>') => scan_chars!(input, '=').then_some(TokenKind::GreaterThanEqual), + Some('<') => match input.next() { + Some('=') => Some(TokenKind::LessThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::LessThan) + } + None => Some(TokenKind::LessThan), + }, + Some('=') => Some(TokenKind::Equal), + Some('>') => match input.next() { + Some('=') => Some(TokenKind::GreaterThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::GreaterThan) + } + None => Some(TokenKind::GreaterThan), + }, + Some('^') => Some(TokenKind::Caret), Some('|') => scan_chars!(input, '|').then_some(TokenKind::BarBar), Some('~') => Some(TokenKind::Tilde), Some(_) => { @@ -8845,11 +8763,6 @@ impl Lexer for Language { longest_match! { { AsciiStringLiteral = ascii_string_literal } - { Caret = caret } - { Equal = equal } - { GreaterThan = greater_than } - { LessThan = less_than } - { Minus = minus } { VersionPragmaValue = version_pragma_value } { Identifier = identifier } } diff --git a/crates/solidity/outputs/cargo/tests/src/cst_output/generated/Statements.rs b/crates/solidity/outputs/cargo/tests/src/cst_output/generated/Statements.rs index ea5c100bfe..504fe0157f 100644 --- a/crates/solidity/outputs/cargo/tests/src/cst_output/generated/Statements.rs +++ b/crates/solidity/outputs/cargo/tests/src/cst_output/generated/Statements.rs @@ -3,6 +3,11 @@ use crate::cst_output::runner::run; use anyhow::Result; +#[test] +fn compound_tokens() -> Result<()> { + run("Statements", "compound_tokens") +} + #[test] fn invalid_termination() -> Result<()> { run("Statements", "invalid_termination") diff --git a/crates/solidity/outputs/npm/crate/src/generated/language.rs b/crates/solidity/outputs/npm/crate/src/generated/language.rs index d25802227e..176234241c 100644 --- a/crates/solidity/outputs/npm/crate/src/generated/language.rs +++ b/crates/solidity/outputs/npm/crate/src/generated/language.rs @@ -5594,15 +5594,6 @@ impl Language { * Scanner Functions ********************************************/ - #[allow(unused_assignments, unused_parens)] - fn ampersand(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '&'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '&')) - ) - } - #[allow(unused_assignments, unused_parens)] fn ascii_escape(&self, input: &mut ParserContext) -> bool { scan_choice!( @@ -5627,29 +5618,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn asterisk(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '*'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '*')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn bang(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '!'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn bar(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '|'), - scan_choice!(input, scan_chars!(input, '|'), scan_chars!(input, '=')) - ) - } - #[allow(unused_assignments, unused_parens)] fn bytes_keyword(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -5695,16 +5663,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn caret(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '^'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn colon(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, ':'), scan_chars!(input, '=')) - } - #[allow(unused_assignments, unused_parens)] fn decimal_digits(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -5837,15 +5795,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn equal(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '='), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - #[allow(unused_assignments, unused_parens)] fn escape_sequence(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6142,33 +6091,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>'), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn greater_than_greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>', '>'), - scan_choice!(input, scan_chars!(input, '>'), scan_chars!(input, '=')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn greater_than_greater_than_greater_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '>', '>', '>'), - scan_chars!(input, '=') - ) - } - #[allow(unused_assignments, unused_parens)] fn hex_byte_escape(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6324,34 +6246,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn less_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '<'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '<')) - ) - } - - #[allow(unused_assignments, unused_parens)] - fn less_than_less_than(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '<', '<'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn minus(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '-'), - scan_choice!( - input, - scan_chars!(input, '>'), - scan_chars!(input, '='), - scan_chars!(input, '-') - ) - ) - } - #[allow(unused_assignments, unused_parens)] fn multiline_comment(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6370,20 +6264,6 @@ impl Language { ) } - #[allow(unused_assignments, unused_parens)] - fn percent(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '%'), scan_chars!(input, '=')) - } - - #[allow(unused_assignments, unused_parens)] - fn plus(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!( - input, - scan_chars!(input, '+'), - scan_choice!(input, scan_chars!(input, '='), scan_chars!(input, '+')) - ) - } - #[allow(unused_assignments, unused_parens)] fn raw_identifier(&self, input: &mut ParserContext) -> bool { scan_sequence!( @@ -6447,11 +6327,6 @@ impl Language { } } - #[allow(unused_assignments, unused_parens)] - fn slash(&self, input: &mut ParserContext) -> bool { - scan_not_followed_by!(input, scan_chars!(input, '/'), scan_chars!(input, '=')) - } - #[allow(unused_assignments, unused_parens)] fn ufixed_keyword(&self, input: &mut ParserContext) -> bool { scan_choice!( @@ -8644,16 +8519,30 @@ impl Lexer for Language { input.set_position(save); if let Some(kind) = match input.next() { - Some('!') => scan_chars!(input, '=').then_some(TokenKind::BangEqual), - Some('%') => scan_chars!(input, '=').then_some(TokenKind::PercentEqual), + Some('!') => match input.next() { + Some('=') => Some(TokenKind::BangEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Bang) + } + None => Some(TokenKind::Bang), + }, + Some('%') => match input.next() { + Some('=') => Some(TokenKind::PercentEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Percent) + } + None => Some(TokenKind::Percent), + }, Some('&') => match input.next() { Some('&') => Some(TokenKind::AmpersandAmpersand), Some('=') => Some(TokenKind::AmpersandEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Ampersand) } - None => None, + None => Some(TokenKind::Ampersand), }, Some('(') => Some(TokenKind::OpenParen), Some(')') => Some(TokenKind::CloseParen), @@ -8662,18 +8551,18 @@ impl Lexer for Language { Some('=') => Some(TokenKind::AsteriskEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Asterisk) } - None => None, + None => Some(TokenKind::Asterisk), }, Some('+') => match input.next() { Some('+') => Some(TokenKind::PlusPlus), Some('=') => Some(TokenKind::PlusEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Plus) } - None => None, + None => Some(TokenKind::Plus), }, Some(',') => Some(TokenKind::Comma), Some('-') => match input.next() { @@ -8681,64 +8570,92 @@ impl Lexer for Language { Some('=') => Some(TokenKind::MinusEqual), Some(_) => { input.undo(); - None + Some(TokenKind::Minus) } - None => None, + None => Some(TokenKind::Minus), }, Some('.') => Some(TokenKind::Period), - Some('/') => scan_chars!(input, '=').then_some(TokenKind::SlashEqual), + Some('/') => match input.next() { + Some('=') => Some(TokenKind::SlashEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Slash) + } + None => Some(TokenKind::Slash), + }, + Some(':') => Some(TokenKind::Colon), Some(';') => Some(TokenKind::Semicolon), Some('<') => match input.next() { - Some('<') => { - scan_chars!(input, '=').then_some(TokenKind::LessThanLessThanEqual) - } + Some('<') => match input.next() { + Some('=') => Some(TokenKind::LessThanLessThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::LessThanLessThan) + } + None => Some(TokenKind::LessThanLessThan), + }, Some('=') => Some(TokenKind::LessThanEqual), Some(_) => { input.undo(); - None + Some(TokenKind::LessThan) } - None => None, + None => Some(TokenKind::LessThan), }, Some('=') => match input.next() { Some('=') => Some(TokenKind::EqualEqual), Some('>') => Some(TokenKind::EqualGreaterThan), Some(_) => { input.undo(); - None + Some(TokenKind::Equal) } - None => None, + None => Some(TokenKind::Equal), }, Some('>') => match input.next() { Some('=') => Some(TokenKind::GreaterThanEqual), Some('>') => match input.next() { Some('=') => Some(TokenKind::GreaterThanGreaterThanEqual), - Some('>') => scan_chars!(input, '=') - .then_some(TokenKind::GreaterThanGreaterThanGreaterThanEqual), + Some('>') => match input.next() { + Some('=') => { + Some(TokenKind::GreaterThanGreaterThanGreaterThanEqual) + } + Some(_) => { + input.undo(); + Some(TokenKind::GreaterThanGreaterThanGreaterThan) + } + None => Some(TokenKind::GreaterThanGreaterThanGreaterThan), + }, Some(_) => { input.undo(); - None + Some(TokenKind::GreaterThanGreaterThan) } - None => None, + None => Some(TokenKind::GreaterThanGreaterThan), }, Some(_) => { input.undo(); - None + Some(TokenKind::GreaterThan) } - None => None, + None => Some(TokenKind::GreaterThan), }, Some('?') => Some(TokenKind::QuestionMark), Some('[') => Some(TokenKind::OpenBracket), Some(']') => Some(TokenKind::CloseBracket), - Some('^') => scan_chars!(input, '=').then_some(TokenKind::CaretEqual), + Some('^') => match input.next() { + Some('=') => Some(TokenKind::CaretEqual), + Some(_) => { + input.undo(); + Some(TokenKind::Caret) + } + None => Some(TokenKind::Caret), + }, Some('{') => Some(TokenKind::OpenBrace), Some('|') => match input.next() { Some('=') => Some(TokenKind::BarEqual), Some('|') => Some(TokenKind::BarBar), Some(_) => { input.undo(); - None + Some(TokenKind::Bar) } - None => None, + None => Some(TokenKind::Bar), }, Some('}') => Some(TokenKind::CloseBrace), Some('~') => Some(TokenKind::Tilde), @@ -8754,32 +8671,16 @@ impl Lexer for Language { input.set_position(save); longest_match! { - { Ampersand = ampersand } { AsciiStringLiteral = ascii_string_literal } - { Asterisk = asterisk } - { Bang = bang } - { Bar = bar } { BytesKeyword = bytes_keyword } - { Caret = caret } - { Colon = colon } { DecimalLiteral = decimal_literal } { EndOfLine = end_of_line } - { Equal = equal } { FixedKeyword = fixed_keyword } - { GreaterThan = greater_than } - { GreaterThanGreaterThan = greater_than_greater_than } - { GreaterThanGreaterThanGreaterThan = greater_than_greater_than_greater_than } { HexLiteral = hex_literal } { HexStringLiteral = hex_string_literal } { IntKeyword = int_keyword } - { LessThan = less_than } - { LessThanLessThan = less_than_less_than } - { Minus = minus } { MultilineComment = multiline_comment } - { Percent = percent } - { Plus = plus } { SingleLineComment = single_line_comment } - { Slash = slash } { UfixedKeyword = ufixed_keyword } { UintKeyword = uint_keyword } { UnicodeStringLiteral = unicode_string_literal } @@ -8826,10 +8727,27 @@ impl Lexer for Language { input.set_position(save); if let Some(kind) = match input.next() { + Some('-') => Some(TokenKind::Minus), Some('.') => Some(TokenKind::Period), Some(';') => Some(TokenKind::Semicolon), - Some('<') => scan_chars!(input, '=').then_some(TokenKind::LessThanEqual), - Some('>') => scan_chars!(input, '=').then_some(TokenKind::GreaterThanEqual), + Some('<') => match input.next() { + Some('=') => Some(TokenKind::LessThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::LessThan) + } + None => Some(TokenKind::LessThan), + }, + Some('=') => Some(TokenKind::Equal), + Some('>') => match input.next() { + Some('=') => Some(TokenKind::GreaterThanEqual), + Some(_) => { + input.undo(); + Some(TokenKind::GreaterThan) + } + None => Some(TokenKind::GreaterThan), + }, + Some('^') => Some(TokenKind::Caret), Some('|') => scan_chars!(input, '|').then_some(TokenKind::BarBar), Some('~') => Some(TokenKind::Tilde), Some(_) => { @@ -8845,11 +8763,6 @@ impl Lexer for Language { longest_match! { { AsciiStringLiteral = ascii_string_literal } - { Caret = caret } - { Equal = equal } - { GreaterThan = greater_than } - { LessThan = less_than } - { Minus = minus } { VersionPragmaValue = version_pragma_value } { Identifier = identifier } } diff --git a/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/generated/0.4.11-success.yml b/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/generated/0.4.11-success.yml new file mode 100644 index 0000000000..7726a65ce5 --- /dev/null +++ b/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/generated/0.4.11-success.yml @@ -0,0 +1,138 @@ +# This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +Source: > + 1 │ a && b; │ 0..7 + 2 │ a &= b; │ 8..15 + 3 │ a || b; │ 16..23 + 4 │ a |= b; │ 24..31 + 5 │ a / b; │ 32..38 + 6 │ a /= b; │ 39..46 + 7 │ a > b; │ 47..53 + 8 │ a >= b; │ 54..61 + 9 │ a >> b; │ 62..69 + 10 │ a >>= b; │ 70..78 + 11 │ a >>>= b; │ 79..88 + 12 │ a << b; │ 89..96 + 13 │ a <<= b; │ 97..105 + +Errors: [] + +Tree: + - Statements (Rule): # 0..106 "a && b;\na &= b;\na || b;\na |= b;\na / b;\na /= b;\na >..." + - ExpressionStatement (Rule): # 0..8 "a && b;\n" + - Expression (Rule): # 0..6 "a && b" + - BinaryExpression (Rule): # 0..6 "a && b" + - Expression (Rule): # 0..1 "a" + - Identifier (Token): "a" # 0..1 + - AmpersandAmpersand (Token): "&&" # 2..4 + - Expression (Rule): # 4..6 " b" + - Identifier (Token): "b" # 5..6 + - Semicolon (Token): ";" # 6..7 + - ExpressionStatement (Rule): # 8..16 "a &= b;\n" + - Expression (Rule): # 8..14 "a &= b" + - BinaryExpression (Rule): # 8..14 "a &= b" + - Expression (Rule): # 8..9 "a" + - Identifier (Token): "a" # 8..9 + - AmpersandEqual (Token): "&=" # 10..12 + - Expression (Rule): # 12..14 " b" + - Identifier (Token): "b" # 13..14 + - Semicolon (Token): ";" # 14..15 + - ExpressionStatement (Rule): # 16..24 "a || b;\n" + - Expression (Rule): # 16..22 "a || b" + - BinaryExpression (Rule): # 16..22 "a || b" + - Expression (Rule): # 16..17 "a" + - Identifier (Token): "a" # 16..17 + - BarBar (Token): "||" # 18..20 + - Expression (Rule): # 20..22 " b" + - Identifier (Token): "b" # 21..22 + - Semicolon (Token): ";" # 22..23 + - ExpressionStatement (Rule): # 24..32 "a |= b;\n" + - Expression (Rule): # 24..30 "a |= b" + - BinaryExpression (Rule): # 24..30 "a |= b" + - Expression (Rule): # 24..25 "a" + - Identifier (Token): "a" # 24..25 + - BarEqual (Token): "|=" # 26..28 + - Expression (Rule): # 28..30 " b" + - Identifier (Token): "b" # 29..30 + - Semicolon (Token): ";" # 30..31 + - ExpressionStatement (Rule): # 32..39 "a / b;\n" + - Expression (Rule): # 32..37 "a / b" + - BinaryExpression (Rule): # 32..37 "a / b" + - Expression (Rule): # 32..33 "a" + - Identifier (Token): "a" # 32..33 + - Slash (Token): "/" # 34..35 + - Expression (Rule): # 35..37 " b" + - Identifier (Token): "b" # 36..37 + - Semicolon (Token): ";" # 37..38 + - ExpressionStatement (Rule): # 39..47 "a /= b;\n" + - Expression (Rule): # 39..45 "a /= b" + - BinaryExpression (Rule): # 39..45 "a /= b" + - Expression (Rule): # 39..40 "a" + - Identifier (Token): "a" # 39..40 + - SlashEqual (Token): "/=" # 41..43 + - Expression (Rule): # 43..45 " b" + - Identifier (Token): "b" # 44..45 + - Semicolon (Token): ";" # 45..46 + - ExpressionStatement (Rule): # 47..54 "a > b;\n" + - Expression (Rule): # 47..52 "a > b" + - BinaryExpression (Rule): # 47..52 "a > b" + - Expression (Rule): # 47..48 "a" + - Identifier (Token): "a" # 47..48 + - GreaterThan (Token): ">" # 49..50 + - Expression (Rule): # 50..52 " b" + - Identifier (Token): "b" # 51..52 + - Semicolon (Token): ";" # 52..53 + - ExpressionStatement (Rule): # 54..62 "a >= b;\n" + - Expression (Rule): # 54..60 "a >= b" + - BinaryExpression (Rule): # 54..60 "a >= b" + - Expression (Rule): # 54..55 "a" + - Identifier (Token): "a" # 54..55 + - GreaterThanEqual (Token): ">=" # 56..58 + - Expression (Rule): # 58..60 " b" + - Identifier (Token): "b" # 59..60 + - Semicolon (Token): ";" # 60..61 + - ExpressionStatement (Rule): # 62..70 "a >> b;\n" + - Expression (Rule): # 62..68 "a >> b" + - BinaryExpression (Rule): # 62..68 "a >> b" + - Expression (Rule): # 62..63 "a" + - Identifier (Token): "a" # 62..63 + - GreaterThanGreaterThan (Token): ">>" # 64..66 + - Expression (Rule): # 66..68 " b" + - Identifier (Token): "b" # 67..68 + - Semicolon (Token): ";" # 68..69 + - ExpressionStatement (Rule): # 70..79 "a >>= b;\n" + - Expression (Rule): # 70..77 "a >>= b" + - BinaryExpression (Rule): # 70..77 "a >>= b" + - Expression (Rule): # 70..71 "a" + - Identifier (Token): "a" # 70..71 + - GreaterThanGreaterThanEqual (Token): ">>=" # 72..75 + - Expression (Rule): # 75..77 " b" + - Identifier (Token): "b" # 76..77 + - Semicolon (Token): ";" # 77..78 + - ExpressionStatement (Rule): # 79..89 "a >>>= b;\n" + - Expression (Rule): # 79..87 "a >>>= b" + - BinaryExpression (Rule): # 79..87 "a >>>= b" + - Expression (Rule): # 79..80 "a" + - Identifier (Token): "a" # 79..80 + - GreaterThanGreaterThanGreaterThanEqual (Token): ">>>=" # 81..85 + - Expression (Rule): # 85..87 " b" + - Identifier (Token): "b" # 86..87 + - Semicolon (Token): ";" # 87..88 + - ExpressionStatement (Rule): # 89..97 "a << b;\n" + - Expression (Rule): # 89..95 "a << b" + - BinaryExpression (Rule): # 89..95 "a << b" + - Expression (Rule): # 89..90 "a" + - Identifier (Token): "a" # 89..90 + - LessThanLessThan (Token): "<<" # 91..93 + - Expression (Rule): # 93..95 " b" + - Identifier (Token): "b" # 94..95 + - Semicolon (Token): ";" # 95..96 + - ExpressionStatement (Rule): # 97..106 "a <<= b;\n" + - Expression (Rule): # 97..104 "a <<= b" + - BinaryExpression (Rule): # 97..104 "a <<= b" + - Expression (Rule): # 97..98 "a" + - Identifier (Token): "a" # 97..98 + - LessThanLessThanEqual (Token): "<<=" # 99..102 + - Expression (Rule): # 102..104 " b" + - Identifier (Token): "b" # 103..104 + - Semicolon (Token): ";" # 104..105 diff --git a/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/input.sol b/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/input.sol new file mode 100644 index 0000000000..25e89b9a03 --- /dev/null +++ b/crates/solidity/testing/snapshots/cst_output/Statements/compound_tokens/input.sol @@ -0,0 +1,13 @@ +a && b; +a &= b; +a || b; +a |= b; +a / b; +a /= b; +a > b; +a >= b; +a >> b; +a >>= b; +a >>>= b; +a << b; +a <<= b;