Co-locate parser generator logic (#972)

Part of #638, removes the `codegen_grammar` crate

Instead of having the definitions and the parser generation separate, we
now co-locate this and provide a parser-specific model (for MVC
rendering introduced in #962) for the code generation. This mostly moves
code around and documents a bit more what we already have.

This is fairly non-controversial and I am for #638 to be incremental, so
this is meant to progress the issue rather than fix it in one giant PR
to keep up the velocity and land it as soon as possible, accounting for
my upcoming move to Hardhat soon. If this generally makes sense, I'd
love to have a rubber stamp for it; for now, I'll work on top of this to
further migrate off DSLv1 definitions.

Cargo.toml

@@ -8,7 +8,6 @@ publish = false
 resolver = "2"
 members = [
     "crates/codegen/ebnf",
-    "crates/codegen/grammar",
     "crates/codegen/language/definition",
     "crates/codegen/language/internal_macros",
     "crates/codegen/language/macros",
@@ -47,7 +46,6 @@ members = [
 # Internal
 #
 codegen_ebnf = { path = "crates/codegen/ebnf" }
-codegen_grammar = { path = "crates/codegen/grammar" }
 codegen_language_definition = { path = "crates/codegen/language/definition" }
 codegen_language_internal_macros = { path = "crates/codegen/language/internal_macros" }
 codegen_language_macros = { path = "crates/codegen/language/macros" }

crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2

@@ -21,7 +21,7 @@ pub enum NonTerminalKind {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for variant in model.nonterminal_kinds -%}
+        {%- for variant in model.parser.nonterminal_kinds -%}
             {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
             {{ variant }},
         {%- endfor -%}
@@ -63,7 +63,7 @@ pub enum EdgeLabel {
         Stub2,
         Stub3,
     {%- else -%}
-        {% for variant in model.labels -%}
+        {% for variant in model.parser.labels -%}
             {{ variant | pascal_case }},
         {%- endfor -%}
     {%- endif -%}
@@ -95,7 +95,7 @@ pub enum TerminalKind {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for variant in model.terminal_kinds -%}
+        {%- for variant in model.parser.terminal_kinds -%}
             {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
             {{ variant }},
         {%- endfor -%}
@@ -109,7 +109,7 @@ impl metaslang_cst::TerminalKind for TerminalKind {
         {%- else -%}
             matches!(
                 self,
-                {%- for variant in model.trivia_scanner_names -%}
+                {%- for variant in model.parser.trivia_scanner_names -%}
                     | Self::{{ variant }}
                 {%- endfor -%}
             )
@@ -125,7 +125,7 @@ pub(crate) enum LexicalContext {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for context_name, _ in model.scanner_contexts %}
+        {%- for context_name, _ in model.parser.scanner_contexts %}
             {{ context_name }},
         {%- endfor %}
     {%- endif -%}
@@ -140,7 +140,7 @@ pub(crate) trait IsLexicalContext {
 #[allow(non_snake_case)]
 pub(crate) mod LexicalContextType {
     {%- if not rendering_in_stubs -%}
-        {%- for context_name, _ in model.scanner_contexts %}
+        {%- for context_name, _ in model.parser.scanner_contexts %}
             pub struct {{ context_name }};
 
             impl super::IsLexicalContext for {{ context_name }} {

crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2

@@ -29,7 +29,7 @@ use crate::parser_support::{
 #[cfg_attr(feature = "slang_napi_interfaces", napi(namespace = "language"))]
 pub struct Language {
     {%- if not rendering_in_stubs -%}
-        {%- for version in model.referenced_versions -%}
+        {%- for version in model.parser.referenced_versions -%}
             pub(crate) version_is_at_least_{{ version | replace(from=".", to="_") }}: bool,
         {%- endfor -%}
     {%- endif -%}
@@ -67,7 +67,7 @@ impl Language {
         if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() {
             Ok(Self {
                 {%- if not rendering_in_stubs -%}
-                    {%- for version in model.referenced_versions %}
+                    {%- for version in model.parser.referenced_versions %}
                         version_is_at_least_{{ version | replace(from=".", to="_") }}: Version::new({{ version | split(pat=".") | join(sep=", ") }}) <= version,
                     {%- endfor -%}
                 {%- endif -%}
@@ -89,12 +89,12 @@ impl Language {
         *         Parser Functions
         ********************************************/
 
-        {% for parser_name, parser_code in model.parser_functions %}
+        {% for parser_name, parser_code in model.parser.parser_functions %}
             #[allow(unused_assignments, unused_parens)]
             fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} }
         {% endfor %}
 
-        {% for parser_name, parser_code in model.trivia_parser_functions %}
+        {% for parser_name, parser_code in model.parser.trivia_parser_functions %}
             #[allow(unused_assignments, unused_parens)]
             fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} }
         {% endfor %}
@@ -103,12 +103,12 @@ impl Language {
         *         Scanner Functions
         ********************************************/
 
-        {% for scanner_name, scanner_code in model.scanner_functions %}
+        {% for scanner_name, scanner_code in model.parser.scanner_functions %}
             #[allow(unused_assignments, unused_parens)]
             fn {{ scanner_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> bool { {{ scanner_code }} }
         {% endfor %}
 
-        {%- for keyword_name, keyword_code in model.keyword_compound_scanners %}
+        {%- for keyword_name, keyword_code in model.parser.keyword_compound_scanners %}
             #[inline]
             fn {{ keyword_name | snake_case }}(&self, input: &mut ParserContext<'_>, ident: &str) -> KeywordScan { {{ keyword_code }} }
         {% endfor %}
@@ -120,7 +120,7 @@ impl Language {
             unreachable!("Attempting to parse in stubs: {kind}: {input}")
         {%- else -%}
             match kind {
-                {%- for parser_name, _ in model.parser_functions -%}
+                {%- for parser_name, _ in model.parser.parser_functions -%}
                     NonTerminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input),
                 {%- endfor -%}
             }
@@ -150,7 +150,7 @@ impl Lexer for Language {
             unreachable!("Invoking delimiters in stubs.")
         {%- else -%}
             match LexCtx::value() {
-                {%- for context_name, context in model.scanner_contexts %}
+                {%- for context_name, context in model.parser.scanner_contexts %}
                     LexicalContext::{{ context_name }} => &[
                         {%- for open, close in context.delimiters %}
                             (TerminalKind::{{ open }}, TerminalKind::{{ close }}),
@@ -183,7 +183,7 @@ impl Lexer for Language {
             }
 
             match LexCtx::value() {
-                {%- for context_name, context in model.scanner_contexts %}
+                {%- for context_name, context in model.parser.scanner_contexts %}
                     LexicalContext::{{ context_name }} => {
                         if let Some(kind) = {{ context.literal_scanner }} {
                             furthest_position = input.position();

crates/codegen/runtime/generator/Cargo.toml

@@ -7,7 +7,6 @@ publish = false
 
 [dependencies]
 anyhow = { workspace = true }
-codegen_grammar = { workspace = true }
 codegen_language_definition = { workspace = true }
 indexmap = { workspace = true }
 Inflector = { workspace = true }
@@ -17,6 +16,7 @@ proc-macro2 = { workspace = true }
 quote = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
+strum_macros = { workspace = true }
 
 [lints]
 workspace = true

crates/codegen/runtime/generator/src/lib.rs

@@ -1,5 +1,3 @@
-#![allow(clippy::too_many_lines)]
-
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 
@@ -12,12 +10,8 @@ use serde::Serialize;
 use crate::model::RuntimeModel;
 
 mod ast;
-mod keyword_scanner_definition;
 mod model;
-mod parser_definition;
-mod precedence_parser_definition;
-mod scanner_definition;
-mod trie;
+mod parser;
 
 pub enum OutputLanguage {
     Cargo,