refactor: Simplify peeking significant tokens in recovery

crates/codegen/parser/runtime/src/lexer.rs

@@ -14,15 +14,32 @@ pub trait Lexer {
     #[doc(hidden)]
     fn trailing_trivia(&self, input: &mut ParserContext) -> ParserResult;
     #[doc(hidden)]
+    /// Returns valid grouping delimiters in the given lexical context.
     fn delimiters<LexCtx: IsLexicalContext>() -> &'static [(TokenKind, TokenKind)];
 
+    /// Peeks the next token, including trivia. Does not advance the input.
     fn peek_token<LexCtx: IsLexicalContext>(&self, input: &mut ParserContext) -> Option<TokenKind> {
         let start = input.position();
         let token = self.next_token::<LexCtx>(input);
         input.set_position(start);
         token
     }
 
+    /// Peeks the next significant (i.e. non-trivia) token. Does not advance the input.
+    fn peek_token_with_trivia<LexCtx: IsLexicalContext>(
+        &self,
+        input: &mut ParserContext,
+    ) -> Option<TokenKind> {
+        let start = input.position();
+
+        let _ = self.leading_trivia(input);
+        let token = self.next_token::<LexCtx>(input);
+
+        input.set_position(start);
+        token
+    }
+
+    /// Attempts to consume the next expected token. Advances the input only if the token matches.
     fn parse_token<LexCtx: IsLexicalContext>(
         &self,
         input: &mut ParserContext,
@@ -41,6 +58,8 @@ pub trait Lexer {
         )
     }
 
+    /// Attempts to consume the next significant token including both leading and trailing trivia.
+    /// Advances the input only if the token matches.
     fn parse_token_with_trivia<LexCtx: IsLexicalContext>(
         &self,
         input: &mut ParserContext,

crates/codegen/parser/runtime/src/support/recovery.rs

@@ -49,16 +49,6 @@ impl ParserResult {
     ) -> ParserResult {
         let before_recovery = input.position();
 
-        let mut peek_token_after_trivia = || {
-            let start = input.position();
-
-            opt_parse(input, |input| lexer.leading_trivia(input));
-            let token = lexer.next_token::<LexCtx>(input);
-
-            input.set_position(start);
-            token
-        };
-
         enum ParseResultKind {
             Match,
             Incomplete,
@@ -71,7 +61,9 @@ impl ParserResult {
                 result.expected_tokens,
                 ParseResultKind::Incomplete,
             ),
-            ParserResult::Match(result) if peek_token_after_trivia() != Some(expected) => {
+            ParserResult::Match(result)
+                if lexer.peek_token_with_trivia::<LexCtx>(input) != Some(expected) =>
+            {
                 (result.nodes, result.expected_tokens, ParseResultKind::Match)
             }
             ParserResult::NoMatch(result) if recover_from_no_match.as_bool() => {

crates/codegen/parser/runtime/src/support/separated_helper.rs

@@ -26,16 +26,9 @@ impl SeparatedHelper {
                 ParserResult::Match(r#match) => {
                     accum.extend(r#match.nodes);
 
-                    // Parse the leading trivia so that we can peek the next significant token
-                    if let ParserResult::Match(r#match) = lexer.leading_trivia(input) {
-                        accum.extend(r#match.nodes);
-                    }
-
-                    match lexer.peek_token::<LexCtx>(input) {
+                    match lexer.peek_token_with_trivia::<LexCtx>(input) {
                         Some(token) if token == separator => {
-                            let separator =
-                                lexer.parse_token_with_trivia::<LexCtx>(input, separator);
-                            match separator {
+                            match lexer.parse_token_with_trivia::<LexCtx>(input, separator) {
                                 ParserResult::Match(r#match) => {
                                     accum.extend(r#match.nodes);
                                     continue;

crates/solidity/testing/snapshots/cst_output/ContractDefinition/function_multiple_delimiters/generated/0.4.11-failure.yml

@@ -90,7 +90,7 @@ Tree:
                       - Statement (Rule): # 201..262 "\n    (bool success, ) = recipient.call{ value: amo..."
                           - TupleDeconstructionStatement (Rule): # 201..262 "\n    (bool success, ) = recipient.call{ value: amo..."
                               - OpenParen (Token): "(" # 206..207
-                              - TupleMembersList (Rule): # 207..221 "bool success, "
+                              - TupleMembersList (Rule): # 207..220 "bool success,"
                                   - TupleMember (Rule): # 207..219 "bool success"
                                       - TypeName (Rule): # 207..211 "bool"
                                           - BoolKeyword (Token): "bool" # 207..211

crates/solidity/testing/snapshots/cst_output/ContractDefinition/function_multiple_delimiters/generated/0.6.2-success.yml

@@ -82,7 +82,7 @@ Tree:
                       - Statement (Rule): # 201..262 "\n    (bool success, ) = recipient.call{ value: amo..."
                           - TupleDeconstructionStatement (Rule): # 201..262 "\n    (bool success, ) = recipient.call{ value: amo..."
                               - OpenParen (Token): "(" # 206..207
-                              - TupleMembersList (Rule): # 207..221 "bool success, "
+                              - TupleMembersList (Rule): # 207..220 "bool success,"
                                   - TupleMember (Rule): # 207..219 "bool success"
                                       - TypeName (Rule): # 207..211 "bool"
                                           - BoolKeyword (Token): "bool" # 207..211
@@ -102,7 +102,7 @@ Tree:
                                       - FunctionCallOptions (Rule): # 239..256 "{ value: amount }"
                                           - NamedArgumentsDeclaration (Rule): # 239..256 "{ value: amount }"
                                               - OpenBrace (Token): "{" # 239..240
-                                              - NamedArgumentsList (Rule): # 240..255 " value: amount "
+                                              - NamedArgumentsList (Rule): # 240..254 " value: amount"
                                                   - NamedArgument (Rule): # 240..254 " value: amount"
                                                       - Identifier (Token): "value" # 241..246
                                                       - Colon (Token): ":" # 246..247

crates/solidity/testing/snapshots/cst_output/ContractDefinition/inheritence_specifier/generated/0.4.11-success.yml

@@ -9,9 +9,9 @@ Tree:
   - ContractDefinition (Rule): # 0..41 "contract Sample is Foo, Bar(1, 2), Baz {}"
       - ContractKeyword (Token): "contract" # 0..8
       - Identifier (Token): "Sample" # 9..15
-      - InheritanceSpecifier (Rule): # 15..39 " is Foo, Bar(1, 2), Baz "
+      - InheritanceSpecifier (Rule): # 15..38 " is Foo, Bar(1, 2), Baz"
           - IsKeyword (Token): "is" # 16..18
-          - InheritanceTypesList (Rule): # 18..39 " Foo, Bar(1, 2), Baz "
+          - InheritanceTypesList (Rule): # 18..38 " Foo, Bar(1, 2), Baz"
               - InheritanceType (Rule): # 18..22 " Foo"
                   - IdentifierPath (Rule): # 18..22 " Foo"
                       - Identifier (Token): "Foo" # 19..22
@@ -31,8 +31,8 @@ Tree:
                                   - DecimalLiteral (Token): "2" # 31..32
                       - CloseParen (Token): ")" # 32..33
               - Comma (Token): "," # 33..34
-              - InheritanceType (Rule): # 34..39 " Baz "
-                  - IdentifierPath (Rule): # 34..39 " Baz "
+              - InheritanceType (Rule): # 34..38 " Baz"
+                  - IdentifierPath (Rule): # 34..38 " Baz"
                       - Identifier (Token): "Baz" # 35..38
       - OpenBrace (Token): "{" # 39..40
       - CloseBrace (Token): "}" # 40..41

crates/solidity/testing/snapshots/cst_output/ContractDefinition/member_enum_definition/generated/0.4.11-success.yml

@@ -21,7 +21,7 @@ Tree:
               - EnumKeyword (Token): "enum" # 20..24
               - Identifier (Token): "State" # 25..30
               - OpenBrace (Token): "{" # 31..32
-              - IdentifiersList (Rule): # 33..55 "    A,\n    B,\n    C\n  "
+              - IdentifiersList (Rule): # 33..53 "    A,\n    B,\n    C\n"
                   - Identifier (Token): "A" # 37..38
                   - Comma (Token): "," # 38..39
                   - Identifier (Token): "B" # 44..45

crates/solidity/testing/snapshots/cst_output/ContractDefinition/member_error_definition/generated/0.4.11-failure.yml

@@ -22,8 +22,8 @@ Tree:
       - OpenBrace (Token): "{" # 16..17
       - ContractMembersList (Rule): # 18..36 "  error Error1();\n"
           - StateVariableDefinition (Rule): # 18..36 "  error Error1();\n"
-              - TypeName (Rule): # 18..26 "  error "
-                  - IdentifierPath (Rule): # 18..26 "  error "
+              - TypeName (Rule): # 18..25 "  error"
+                  - IdentifierPath (Rule): # 18..25 "  error"
                       - Identifier (Token): "error" # 20..25
               - Identifier (Token): "Error1" # 26..32
               - SKIPPED (Token): "()" # 32..34

crates/solidity/testing/snapshots/cst_output/ContractDefinition/member_using_directive/generated/0.4.11-success.yml

@@ -15,8 +15,8 @@ Tree:
       - ContractMembersList (Rule): # 18..35 "  using x for *;\n"
           - UsingDirective (Rule): # 18..35 "  using x for *;\n"
               - UsingKeyword (Token): "using" # 20..25
-              - UsingDirectivePath (Rule): # 25..28 " x "
-                  - IdentifierPath (Rule): # 25..28 " x "
+              - UsingDirectivePath (Rule): # 25..27 " x"
+                  - IdentifierPath (Rule): # 25..27 " x"
                       - Identifier (Token): "x" # 26..27
               - ForKeyword (Token): "for" # 28..31
               - Asterisk (Token): "*" # 32..33

crates/solidity/testing/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.4.11-failure.yml

@@ -52,7 +52,7 @@ Tree:
               - Identifier (Token): "_transfer" # 30..39
               - ParametersDeclaration (Rule): # 39..54 "(address while)"
                   - OpenParen (Token): "(" # 39..40
-                  - ParametersList (Rule): # 40..48 "address "
+                  - ParametersList (Rule): # 40..47 "address"
                       - Parameter (Rule): # 40..47 "address"
                           - TypeName (Rule): # 40..47 "address"
                               - AddressType (Rule): # 40..47 "address"

crates/solidity/testing/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.5.3-failure.yml

@@ -52,7 +52,7 @@ Tree:
               - Identifier (Token): "_transfer" # 30..39
               - ParametersDeclaration (Rule): # 39..54 "(address while)"
                   - OpenParen (Token): "(" # 39..40
-                  - ParametersList (Rule): # 40..48 "address "
+                  - ParametersList (Rule): # 40..47 "address"
                       - Parameter (Rule): # 40..47 "address"
                           - TypeName (Rule): # 40..47 "address"
                               - AddressType (Rule): # 40..47 "address"

crates/solidity/testing/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.6.2-failure.yml

@@ -59,7 +59,7 @@ Tree:
               - Identifier (Token): "_transfer" # 30..39
               - ParametersDeclaration (Rule): # 39..54 "(address while)"
                   - OpenParen (Token): "(" # 39..40
-                  - ParametersList (Rule): # 40..48 "address "
+                  - ParametersList (Rule): # 40..47 "address"
                       - Parameter (Rule): # 40..47 "address"
                           - TypeName (Rule): # 40..47 "address"
                               - AddressType (Rule): # 40..47 "address"

crates/solidity/testing/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.7.0-failure.yml

@@ -59,7 +59,7 @@ Tree:
               - Identifier (Token): "_transfer" # 30..39
               - ParametersDeclaration (Rule): # 39..54 "(address while)"
                   - OpenParen (Token): "(" # 39..40
-                  - ParametersList (Rule): # 40..48 "address "
+                  - ParametersList (Rule): # 40..47 "address"
                       - Parameter (Rule): # 40..47 "address"
                           - TypeName (Rule): # 40..47 "address"
                               - AddressType (Rule): # 40..47 "address"