Discern invalid and missing syntax with different terminal kinds (#1013)

Alternative to #969

Closes #835
Closes #507
Closes #700

This implements the idea from #835:
- introduces a new `TerminalKind::MISSING`
- renames `TerminalKind::SKIPPED` to `TerminalKind::UNRECOGNIZED`
- emits `TerminalKind::MISSING` instead of `TerminalKind::UNRECOGNIZED`
when the tree is empty

When writing this, I came to a conclusion that actually using two
distinc terminal kinds might do more harm than good here, see
#835 (comment).

crates/codegen/runtime/cargo/src/runtime/kinds/mod.rs.jinja2

@@ -84,7 +84,8 @@ impl metaslang_cst::EdgeLabel for EdgeLabel {}
 #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))]
 pub enum TerminalKind {
     // Built-in:
-    SKIPPED,
+    UNRECOGNIZED,
+    MISSING,
 
     // Generated:
     {% if rendering_in_stubs -%}
@@ -112,6 +113,10 @@ impl metaslang_cst::TerminalKind for TerminalKind {
             )
         {%- endif -%}
     }
+
+    fn is_valid(&self) -> bool {
+        !matches!(self, Self::UNRECOGNIZED | Self::MISSING)
+    }
 }
 
 /// The lexical context of the scanner.

crates/codegen/runtime/cargo/src/runtime/language/mod.rs.jinja2

@@ -270,7 +270,7 @@ impl Lexer for Language {
                 // Skip a character if possible and if we didn't recognize a terminal
                 None if input.peek().is_some() => {
                     let _ = input.next();
-                    Some(ScannedTerminal::Single(TerminalKind::SKIPPED))
+                    Some(ScannedTerminal::Single(TerminalKind::UNRECOGNIZED))
                 },
                 None => None,
             }

crates/codegen/runtime/cargo/src/runtime/language/parser_support/choice_helper.rs

@@ -1,8 +1,9 @@
 use std::mem;
 use std::ops::ControlFlow;
 
+use metaslang_cst::TerminalKind as _;
+
 use crate::cst;
-use crate::kinds::TerminalKind;
 use crate::language::parser_support::context::{Marker, ParserContext};
 use crate::language::parser_support::ParserResult;
 use crate::parse_error::ParseError;
@@ -139,9 +140,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize {
         .iter()
         .flat_map(|child| child.cursor_with_offset(TextIndex::ZERO))
         .filter_map(|node| match node {
-            cst::Node::Terminal(terminal) if terminal.kind != TerminalKind::SKIPPED => {
-                Some(terminal.text.len())
-            }
+            cst::Node::Terminal(terminal) if terminal.kind.is_valid() => Some(terminal.text.len()),
             _ => None,
         })
         .sum()

crates/codegen/runtime/cargo/src/runtime/language/parser_support/parser_function.rs

@@ -1,5 +1,7 @@
 use std::rc::Rc;
 
+use metaslang_cst::TerminalKind as _;
+
 use crate::cst::{self, Edge};
 use crate::kinds::TerminalKind;
 use crate::language::lexer::Lexer;
@@ -23,6 +25,7 @@ where
     L: Lexer,
     F: Fn(&L, &mut ParserContext<'_>) -> ParserResult,
 {
+    #[allow(clippy::too_many_lines)]
     fn parse(&self, language: &L, input: &str) -> ParseOutput {
         let mut stream = ParserContext::new(input);
         let mut result = self(language, &mut stream);
@@ -58,13 +61,21 @@ where
         match result {
             ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"),
 
-            ParserResult::NoMatch(no_match) => ParseOutput {
-                parse_tree: cst::Node::terminal(TerminalKind::SKIPPED, input.to_string()),
-                errors: vec![ParseError::new(
-                    TextIndex::ZERO..input.into(),
-                    no_match.expected_terminals,
-                )],
-            },
+            ParserResult::NoMatch(no_match) => {
+                let kind = if input.is_empty() {
+                    TerminalKind::MISSING
+                } else {
+                    TerminalKind::UNRECOGNIZED
+                };
+
+                ParseOutput {
+                    parse_tree: cst::Node::terminal(kind, input.to_string()),
+                    errors: vec![ParseError::new(
+                        TextIndex::ZERO..input.into(),
+                        no_match.expected_terminals,
+                    )],
+                }
+            }
             some_match => {
                 let (nodes, expected_terminals) = match some_match {
                     ParserResult::PrattOperatorMatch(..) | ParserResult::NoMatch(..) => {
@@ -105,8 +116,12 @@ where
                     } else {
                         start
                     };
-                    let skipped_node =
-                        cst::Node::terminal(TerminalKind::SKIPPED, input[start.utf8..].to_string());
+                    let kind = if input[start.utf8..].is_empty() {
+                        TerminalKind::MISSING
+                    } else {
+                        TerminalKind::UNRECOGNIZED
+                    };
+                    let skipped_node = cst::Node::terminal(kind, input[start.utf8..].to_string());
                     let mut new_children = topmost_node.children.clone();
                     new_children.push(Edge::anonymous(skipped_node));
 
@@ -121,13 +136,14 @@ where
                     let parse_tree = cst::Node::Nonterminal(topmost_node);
                     let errors = stream.into_errors();
 
-                    // Sanity check: Make sure that succesful parse is equivalent to not having any SKIPPED nodes
+                    // Sanity check: Make sure that succesful parse is equivalent to not having any invalid nodes
                     debug_assert_eq!(
                         errors.is_empty(),
                         parse_tree
                             .cursor_with_offset(TextIndex::ZERO)
                             .all(|node| node
-                                .as_terminal_with_kind(TerminalKind::SKIPPED)
+                                .as_terminal()
+                                .filter(|tok| !tok.kind.is_valid())
                                 .is_none())
                     );
 

crates/codegen/runtime/cargo/src/runtime/language/parser_support/parser_result.rs

@@ -138,7 +138,11 @@ impl Match {
         self.nodes
             .iter()
             .flat_map(|node| node.cursor_with_offset(TextIndex::ZERO))
-            .all(|node| node.as_terminal_with_kind(TerminalKind::SKIPPED).is_none())
+            .all(|node| {
+                node.as_terminal()
+                    .filter(|tok| !tok.kind.is_valid())
+                    .is_none()
+            })
     }
 }
 
@@ -212,9 +216,7 @@ impl IncompleteMatch {
             .flat_map(|node| node.cursor_with_offset(TextIndex::ZERO))
             .try_fold(0u8, |mut acc, node| {
                 match node {
-                    Node::Terminal(tok)
-                        if tok.kind != TerminalKind::SKIPPED && !tok.kind.is_trivia() =>
-                    {
+                    Node::Terminal(tok) if tok.kind.is_valid() && !tok.kind.is_trivia() => {
                         acc += 1;
                     }
                     _ => {}

crates/codegen/runtime/cargo/src/runtime/language/parser_support/separated_helper.rs

@@ -53,8 +53,13 @@ impl SeparatedHelper {
                     match skip_until_with_nested_delims::<_, LexCtx>(input, lexer, separator) {
                         // A separator was found, so we can recover the incomplete match
                         Some((found, skipped_range)) if found == separator => {
+                            let kind = if skipped_range.is_empty() {
+                                TerminalKind::MISSING
+                            } else {
+                                TerminalKind::UNRECOGNIZED
+                            };
                             accum.push(Edge::anonymous(cst::Node::terminal(
-                                TerminalKind::SKIPPED,
+                                kind,
                                 input.content(skipped_range.utf8()),
                             )));
                             input.emit(ParseError {

crates/codegen/runtime/cargo/src/runtime/language/parser_support/sequence_helper.rs

@@ -149,8 +149,13 @@ impl SequenceHelper {
                     });
                     debug_assert_eq!(next_terminal, Ok(Some(running.found)));
 
+                    let kind = if running.skipped.is_empty() {
+                        TerminalKind::MISSING
+                    } else {
+                        TerminalKind::UNRECOGNIZED
+                    };
                     running.nodes.push(Edge::anonymous(cst::Node::terminal(
-                        TerminalKind::SKIPPED,
+                        kind,
                         std::mem::take(&mut running.skipped),
                     )));
                     running.nodes.extend(next.nodes);

crates/metaslang/cst/src/lib.rs

@@ -36,6 +36,11 @@ pub trait TerminalKind: AbstractKind {
     fn is_trivia(&self) -> bool {
         false
     }
+
+    /// Returns whether the terminal is valid, i.e. does not represent missing or invalid syntax.
+    fn is_valid(&self) -> bool {
+        true
+    }
 }
 
 pub trait NonterminalKind: AbstractKind {}