Use TerminalKind::MISSING for empty UNRECOGNIZED kinds

Xanewok · Jun 16, 2024 · f863bc3 · f863bc3
1 parent 4a233c3
commit f863bc3
Show file tree

Hide file tree

Showing 40 changed files with 144 additions and 81 deletions.
diff --git a/crates/codegen/runtime/cargo/src/runtime/language/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/src/runtime/language/parser_support/parser_function.rs
@@ -25,6 +25,7 @@ where
     L: Lexer,
     F: Fn(&L, &mut ParserContext<'_>) -> ParserResult,
 {
+    #[allow(clippy::too_many_lines)]
     fn parse(&self, language: &L, input: &str) -> ParseOutput {
         let mut stream = ParserContext::new(input);
         let mut result = self(language, &mut stream);
@@ -60,13 +61,21 @@ where
         match result {
             ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"),
 
-            ParserResult::NoMatch(no_match) => ParseOutput {
-                parse_tree: cst::Node::terminal(TerminalKind::UNRECOGNIZED, input.to_string()),
-                errors: vec![ParseError::new(
-                    TextIndex::ZERO..input.into(),
-                    no_match.expected_terminals,
-                )],
-            },
+            ParserResult::NoMatch(no_match) => {
+                let kind = if input.is_empty() {
+                    TerminalKind::MISSING
+                } else {
+                    TerminalKind::UNRECOGNIZED
+                };
+
+                ParseOutput {
+                    parse_tree: cst::Node::terminal(kind, input.to_string()),
+                    errors: vec![ParseError::new(
+                        TextIndex::ZERO..input.into(),
+                        no_match.expected_terminals,
+                    )],
+                }
+            }
             some_match => {
                 let (nodes, expected_terminals) = match some_match {
                     ParserResult::PrattOperatorMatch(..) | ParserResult::NoMatch(..) => {
@@ -107,10 +116,12 @@ where
                     } else {
                         start
                     };
-                    let skipped_node = cst::Node::terminal(
-                        TerminalKind::UNRECOGNIZED,
-                        input[start.utf8..].to_string(),
-                    );
+                    let kind = if input[start.utf8..].is_empty() {
+                        TerminalKind::MISSING
+                    } else {
+                        TerminalKind::UNRECOGNIZED
+                    };
+                    let skipped_node = cst::Node::terminal(kind, input[start.utf8..].to_string());
                     let mut new_children = topmost_node.children.clone();
                     new_children.push(Edge::anonymous(skipped_node));
 

diff --git a/crates/codegen/runtime/cargo/src/runtime/language/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/src/runtime/language/parser_support/separated_helper.rs
@@ -53,8 +53,13 @@ impl SeparatedHelper {
                     match skip_until_with_nested_delims::<_, LexCtx>(input, lexer, separator) {
                         // A separator was found, so we can recover the incomplete match
                         Some((found, skipped_range)) if found == separator => {
+                            let kind = if skipped_range.is_empty() {
+                                TerminalKind::MISSING
+                            } else {
+                                TerminalKind::UNRECOGNIZED
+                            };
                             accum.push(Edge::anonymous(cst::Node::terminal(
-                                TerminalKind::UNRECOGNIZED,
+                                kind,
                                 input.content(skipped_range.utf8()),
                             )));
                             input.emit(ParseError {

diff --git a/crates/codegen/runtime/cargo/src/runtime/language/parser_support/sequence_helper.rs b/crates/codegen/runtime/cargo/src/runtime/language/parser_support/sequence_helper.rs
@@ -149,8 +149,13 @@ impl SequenceHelper {
                     });
                     debug_assert_eq!(next_terminal, Ok(Some(running.found)));
 
+                    let kind = if running.skipped.is_empty() {
+                        TerminalKind::MISSING
+                    } else {
+                        TerminalKind::UNRECOGNIZED
+                    };
                     running.nodes.push(Edge::anonymous(cst::Node::terminal(
-                        TerminalKind::UNRECOGNIZED,
+                        kind,
                         std::mem::take(&mut running.skipped),
                     )));
                     running.nodes.extend(next.nodes);

diff --git a/...ity/outputs/cargo/slang_solidity/src/generated/language/parser_support/parser_function.rs b/...ity/outputs/cargo/slang_solidity/src/generated/language/parser_support/parser_function.rs
diff --git a/...ty/outputs/cargo/slang_solidity/src/generated/language/parser_support/separated_helper.rs b/...ty/outputs/cargo/slang_solidity/src/generated/language/parser_support/separated_helper.rs
diff --git a/...ity/outputs/cargo/slang_solidity/src/generated/language/parser_support/sequence_helper.rs b/...ity/outputs/cargo/slang_solidity/src/generated/language/parser_support/sequence_helper.rs
diff --git a/...g/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.4.11-failure.yml b/...g/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.4.11-failure.yml
@@ -29,7 +29,7 @@ Tree:
                   - (leading_trivia꞉ Whitespace): "  " # (16..18)
                   - (item꞉ Identifier): "field" # (18..23)
               - (attributes꞉ StateVariableAttributes): [] # (23..23)
-              - (UNRECOGNIZED): "" # (23..23)
+              - (MISSING): "" # (23..23)
               - (semicolon꞉ Semicolon): ";" # (23..24)
               - (trailing_trivia꞉ EndOfLine): "\n" # (24..25)
       - (close_brace꞉ CloseBrace): "}" # (25..26)

diff --git a/...ng/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.6.0-failure.yml b/...ng/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.6.0-failure.yml
@@ -29,7 +29,7 @@ Tree:
                   - (leading_trivia꞉ Whitespace): "  " # (16..18)
                   - (item꞉ Identifier): "field" # (18..23)
               - (attributes꞉ StateVariableAttributes): [] # (23..23)
-              - (UNRECOGNIZED): "" # (23..23)
+              - (MISSING): "" # (23..23)
               - (semicolon꞉ Semicolon): ";" # (23..24)
               - (trailing_trivia꞉ EndOfLine): "\n" # (24..25)
       - (close_brace꞉ CloseBrace): "}" # (25..26)

diff --git a/...ng/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.6.5-failure.yml b/...ng/snapshots/cst_output/ContractDefinition/missing_field_type/generated/0.6.5-failure.yml
@@ -29,7 +29,7 @@ Tree:
                   - (leading_trivia꞉ Whitespace): "  " # (16..18)
                   - (item꞉ Identifier): "field" # (18..23)
               - (attributes꞉ StateVariableAttributes): [] # (23..23)
-              - (UNRECOGNIZED): "" # (23..23)
+              - (MISSING): "" # (23..23)
               - (semicolon꞉ Semicolon): ";" # (23..24)
               - (trailing_trivia꞉ EndOfLine): "\n" # (24..25)
       - (close_brace꞉ CloseBrace): "}" # (25..26)

diff --git a/...ting/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.8.0-failure.yml b/...ting/snapshots/cst_output/ContractDefinition/recovery_testbed/generated/0.8.0-failure.yml
@@ -146,7 +146,7 @@ Tree:
                                       - (leading_trivia꞉ Whitespace): " " # (195..196)
                                       - (name꞉ Identifier): "sequence" # (196..204)
                                       - (leading_trivia꞉ Whitespace): " " # (204..205)
-                              - (UNRECOGNIZED): "" # (205..205)
+                              - (MISSING): "" # (205..205)
                               - (close_brace꞉ CloseBrace): "}" # (205..206)
                               - (trailing_trivia꞉ EndOfLine): "\n" # (206..207)
                   - (leading_trivia꞉ Whitespace): "\t" # (207..208)

diff --git a/...ng/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.4.11-failure.yml b/...ng/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.4.11-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ng/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.4.22-failure.yml b/...ng/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.4.22-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.6.0-failure.yml b/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.6.0-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.0-failure.yml b/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.0-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.4-failure.yml b/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.4-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.8-failure.yml b/...ing/snapshots/cst_output/ContractDefinition/unterminated_body/generated/0.8.8-failure.yml
@@ -21,4 +21,4 @@ Tree:
       - (leading_trivia꞉ Whitespace): " " # (15..16)
       - (open_brace꞉ OpenBrace): "{" # (16..17)
       - (members꞉ ContractMembers): [] # (17..17)
-      - (UNRECOGNIZED): "" # (17..17)
+      - (MISSING): "" # (17..17)
diff --git a/...ng/snapshots/cst_output/ContractDefinition/zero_length_input/generated/0.4.11-failure.yml b/...ng/snapshots/cst_output/ContractDefinition/zero_length_input/generated/0.4.11-failure.yml
@@ -8,4 +8,4 @@ Errors: # 1 total
        ─[crates/solidity/testing/snapshots/cst_output/ContractDefinition/zero_length_input/input.sol:0:0]
 
 Tree:
-  - (UNRECOGNIZED): "" # (0..0)
+  - (MISSING): "" # (0..0)
diff --git a/...ting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.4.11-failure.yml b/...ting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.4.11-failure.yml
@@ -283,7 +283,7 @@ Tree:
                           - (item꞉ Expression): # "some." (522..527)
                               - (variant꞉ Identifier): "some" # (522..526)
                               - (period꞉ Period): "." # (526..527)
-                          - (UNRECOGNIZED): "" # (527..527)
+                          - (MISSING): "" # (527..527)
                           - (Comma): "," # (527..528)
                           - (item꞉ Expression) ► (variant꞉ MemberAccessExpression): # " next.arg" (528..537)
                               - (operand꞉ Expression): # " next" (528..533)

diff --git a/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.6.0-failure.yml b/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.6.0-failure.yml
@@ -278,7 +278,7 @@ Tree:
                           - (item꞉ IdentifierPath): # "some." (522..527)
                               - (item꞉ Identifier): "some" # (522..526)
                               - (separator꞉ Period): "." # (526..527)
-                          - (UNRECOGNIZED): "" # (527..527)
+                          - (MISSING): "" # (527..527)
                           - (Comma): "," # (527..528)
                           - (item꞉ IdentifierPath): # " next.arg" (528..537)
                               - (leading_trivia꞉ Whitespace): " " # (528..529)

diff --git a/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.6.2-failure.yml b/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.6.2-failure.yml
@@ -201,7 +201,7 @@ Tree:
                                   - (item꞉ NamedArgument): # " no_semicolon" (208..221)
                                       - (leading_trivia꞉ Whitespace): " " # (208..209)
                                       - (name꞉ Identifier): "no_semicolon" # (209..221)
-                                  - (UNRECOGNIZED): "" # (221..221)
+                                  - (MISSING): "" # (221..221)
                                   - (Comma): "," # (221..222)
                               - (leading_trivia꞉ Whitespace): " " # (222..223)
                               - (UNRECOGNIZED): ", " # (223..225)
@@ -338,7 +338,7 @@ Tree:
                           - (item꞉ IdentifierPath): # "some." (522..527)
                               - (item꞉ Identifier): "some" # (522..526)
                               - (separator꞉ Period): "." # (526..527)
-                          - (UNRECOGNIZED): "" # (527..527)
+                          - (MISSING): "" # (527..527)
                           - (Comma): "," # (527..528)
                           - (item꞉ IdentifierPath): # " next.arg" (528..537)
                               - (leading_trivia꞉ Whitespace): " " # (528..529)

diff --git a/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.7.0-failure.yml b/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.7.0-failure.yml
@@ -201,7 +201,7 @@ Tree:
                                   - (item꞉ NamedArgument): # " no_semicolon" (208..221)
                                       - (leading_trivia꞉ Whitespace): " " # (208..209)
                                       - (name꞉ Identifier): "no_semicolon" # (209..221)
-                                  - (UNRECOGNIZED): "" # (221..221)
+                                  - (MISSING): "" # (221..221)
                                   - (Comma): "," # (221..222)
                               - (leading_trivia꞉ Whitespace): " " # (222..223)
                               - (UNRECOGNIZED): ", " # (223..225)
@@ -338,7 +338,7 @@ Tree:
                           - (item꞉ IdentifierPath): # "some." (522..527)
                               - (item꞉ Identifier): "some" # (522..526)
                               - (separator꞉ Period): "." # (526..527)
-                          - (UNRECOGNIZED): "" # (527..527)
+                          - (MISSING): "" # (527..527)
                           - (Comma): "," # (527..528)
                           - (item꞉ IdentifierPath): # " next.arg" (528..537)
                               - (leading_trivia꞉ Whitespace): " " # (528..529)

diff --git a/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.8.0-failure.yml b/...sting/snapshots/cst_output/ContractMembers/separated_recovery/generated/0.8.0-failure.yml
@@ -201,7 +201,7 @@ Tree:
                                   - (item꞉ NamedArgument): # " no_semicolon" (208..221)
                                       - (leading_trivia꞉ Whitespace): " " # (208..209)
                                       - (name꞉ Identifier): "no_semicolon" # (209..221)
-                                  - (UNRECOGNIZED): "" # (221..221)
+                                  - (MISSING): "" # (221..221)
                                   - (Comma): "," # (221..222)
                               - (leading_trivia꞉ Whitespace): " " # (222..223)
                               - (UNRECOGNIZED): ", " # (223..225)
@@ -338,7 +338,7 @@ Tree:
                           - (item꞉ IdentifierPath): # "some." (522..527)
                               - (item꞉ Identifier): "some" # (522..526)
                               - (separator꞉ Period): "." # (526..527)
-                          - (UNRECOGNIZED): "" # (527..527)
+                          - (MISSING): "" # (527..527)
                           - (Comma): "," # (527..528)
                           - (item꞉ IdentifierPath): # " next.arg" (528..537)
                               - (leading_trivia꞉ Whitespace): " " # (528..529)

diff --git a/...idity/testing/snapshots/cst_output/EventDefinition/no_parens/generated/0.4.11-failure.yml b/...idity/testing/snapshots/cst_output/EventDefinition/no_parens/generated/0.4.11-failure.yml
@@ -18,6 +18,6 @@ Tree:
       - (event_keyword꞉ EventKeyword): "event" # (0..5)
       - (leading_trivia꞉ Whitespace): " " # (5..6)
       - (name꞉ Identifier): "MyEvent" # (6..13)
-      - (UNRECOGNIZED): "" # (13..13)
+      - (MISSING): "" # (13..13)
       - (semicolon꞉ Semicolon): ";" # (13..14)
       - (trailing_trivia꞉ EndOfLine): "\n" # (14..15)
diff --git a/...y/testing/snapshots/cst_output/Expression/incomplete_operand/generated/0.4.11-failure.yml b/...y/testing/snapshots/cst_output/Expression/incomplete_operand/generated/0.4.11-failure.yml
@@ -23,4 +23,4 @@ Tree:
           - (leading_trivia꞉ Whitespace): " " # (3..4)
           - (new_keyword꞉ NewKeyword): "new" # (4..7)
           - (trailing_trivia꞉ EndOfLine): "\n" # (7..8)
-      - (UNRECOGNIZED): "" # (8..8)
+      - (MISSING): "" # (8..8)
diff --git a/...ty/testing/snapshots/cst_output/Expression/incomplete_operand/generated/0.8.0-failure.yml b/...ty/testing/snapshots/cst_output/Expression/incomplete_operand/generated/0.8.0-failure.yml
@@ -23,4 +23,4 @@ Tree:
           - (leading_trivia꞉ Whitespace): " " # (3..4)
           - (new_keyword꞉ NewKeyword): "new" # (4..7)
           - (trailing_trivia꞉ EndOfLine): "\n" # (7..8)
-      - (UNRECOGNIZED): "" # (8..8)
+      - (MISSING): "" # (8..8)
diff --git a/...napshots/cst_output/ImportDirective/destructure_import_empty/generated/0.4.11-failure.yml b/...napshots/cst_output/ImportDirective/destructure_import_empty/generated/0.4.11-failure.yml
@@ -22,7 +22,7 @@ Tree:
       - (clause꞉ ImportClause) ► (variant꞉ ImportDeconstruction): # ' {} from "foo"' (37..51)
           - (leading_trivia꞉ Whitespace): " " # (37..38)
           - (open_brace꞉ OpenBrace): "{" # (38..39)
-          - (UNRECOGNIZED): "" # (39..39)
+          - (MISSING): "" # (39..39)
           - (close_brace꞉ CloseBrace): "}" # (39..40)
           - (leading_trivia꞉ Whitespace): " " # (40..41)
           - (from_keyword꞉ FromKeyword): "from" # (41..45)

diff --git a/...y/testing/snapshots/cst_output/SourceUnit/partial_definition/generated/0.4.11-failure.yml b/...y/testing/snapshots/cst_output/SourceUnit/partial_definition/generated/0.4.11-failure.yml
@@ -28,4 +28,4 @@ Tree:
                   - (item꞉ ContractMember) ► (variant꞉ FunctionDefinition): # "  function" (18..28)
                       - (leading_trivia꞉ Whitespace): "  " # (18..20)
                       - (function_keyword꞉ FunctionKeyword): "function" # (20..28)
-      - (UNRECOGNIZED): "" # (28..28)
+      - (MISSING): "" # (28..28)
diff --git a/...cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.4.11-failure.yml b/...cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.4.11-failure.yml
@@ -27,7 +27,7 @@ Tree:
           - (for_keyword꞉ ForKeyword): "for" # (0..3)
           - (open_paren꞉ OpenParen): "(" # (3..4)
           - (initialization꞉ ForStatementInitialization) ► (variant꞉ ExpressionStatement) ► (expression꞉ Expression) ► (variant꞉ ElementaryType) ► (variant꞉ IntKeyword): "int" # (4..7)
-          - (UNRECOGNIZED): "" # (7..7)
+          - (MISSING): "" # (7..7)
           - (close_paren꞉ CloseParen): ")" # (7..8)
           - (trailing_trivia꞉ EndOfLine): "\n" # (8..9)
-      - (UNRECOGNIZED): "" # (9..9)
+      - (MISSING): "" # (9..9)
diff --git a/...cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.4.21-failure.yml b/...cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.4.21-failure.yml
@@ -27,7 +27,7 @@ Tree:
           - (for_keyword꞉ ForKeyword): "for" # (0..3)
           - (open_paren꞉ OpenParen): "(" # (3..4)
           - (initialization꞉ ForStatementInitialization) ► (variant꞉ ExpressionStatement) ► (expression꞉ Expression) ► (variant꞉ ElementaryType) ► (variant꞉ IntKeyword): "int" # (4..7)
-          - (UNRECOGNIZED): "" # (7..7)
+          - (MISSING): "" # (7..7)
           - (close_paren꞉ CloseParen): ")" # (7..8)
           - (trailing_trivia꞉ EndOfLine): "\n" # (8..9)
-      - (UNRECOGNIZED): "" # (9..9)
+      - (MISSING): "" # (9..9)
diff --git a/.../cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.5.0-failure.yml b/.../cst_output/Statements/recovery_ignore_multiple_empty_matches/generated/0.5.0-failure.yml
@@ -27,7 +27,7 @@ Tree:
           - (for_keyword꞉ ForKeyword): "for" # (0..3)
           - (open_paren꞉ OpenParen): "(" # (3..4)
           - (initialization꞉ ForStatementInitialization) ► (variant꞉ ExpressionStatement) ► (expression꞉ Expression) ► (variant꞉ ElementaryType) ► (variant꞉ IntKeyword): "int" # (4..7)
-          - (UNRECOGNIZED): "" # (7..7)
+          - (MISSING): "" # (7..7)
           - (close_paren꞉ CloseParen): ")" # (7..8)
           - (trailing_trivia꞉ EndOfLine): "\n" # (8..9)
-      - (UNRECOGNIZED): "" # (9..9)
+      - (MISSING): "" # (9..9)