Add EBNF grammar as doc comments for TerminalKind and NonterminalKind (…

…#1169) When generating documentation for `TerminalKind` and `NonterminalKind`, generate the EBNF grammar for the current variant and include that in the final output. This affects the Wit output and Rust output (using templates `cst.wit.jinja2` and `(terminal|nonterminal)_kind.rs.jinja2`). * [x] Depends on #1157 * [x] Depends on NomicFoundation/jco#3 Closes #1165
NomicFoundation · Dec 12, 2024 · 8eea9e5 · 8eea9e5
1 parent 6102886
commit 8eea9e5
Show file tree

Hide file tree

Showing 17 changed files with 12,593 additions and 17 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/nonterminal_kind.rs b/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/nonterminal_kind.rs
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/terminal_kind.rs b/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/terminal_kind.rs
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/nonterminal_kind.rs.jinja2 b/crates/codegen/runtime/cargo/crate/src/runtime/cst/nonterminal_kind.rs.jinja2
@@ -1,4 +1,7 @@
 {# This needs to stay in sync with the wit-bindgen output #}
+#[allow(clippy::doc_markdown)]
+#[allow(clippy::doc_link_with_quotes)]
+
 {% if model.kinds.nonterminal_kinds|length <= 256 %} #[repr(u8)] {% else %} #[repr(u16)] {% endif %}
 #[derive(
     Debug,
@@ -21,8 +24,10 @@ pub enum NonterminalKind {
         Stub3,
     {%- else -%}
         {%- for variant in model.kinds.nonterminal_kinds -%}
-            {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
-            {{ variant }},
+            /// ```ebnf
+            {{ variant.documentation | indent(prefix = "/// ", first = true, blank = true) }}
+            /// ```
+            {{ variant.id }},
         {%- endfor -%}
     {%- endif -%}
 }

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/terminal_kind.rs.jinja2 b/crates/codegen/runtime/cargo/crate/src/runtime/cst/terminal_kind.rs.jinja2
@@ -15,6 +15,8 @@
 )]
 #[derive(Clone, Copy)] 
 #[allow(clippy::upper_case_acronyms)]
+#[allow(clippy::doc_markdown)]
+#[allow(clippy::doc_link_with_quotes)]
 pub enum TerminalKind {
     // Built-in:
     UNRECOGNIZED,
@@ -27,8 +29,10 @@ pub enum TerminalKind {
         Stub3,
     {%- else -%}
         {%- for variant in model.kinds.terminal_kinds -%}
-            {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
-            {{ variant }},
+            /// ```ebnf
+            {{ variant.documentation | indent(prefix = "/// ", first = true, blank = true) }}
+            /// ```
+            {{ variant.id }},
         {%- endfor -%}
     {%- endif -%}
 }

diff --git a/crates/codegen/runtime/cargo/wasm/src/runtime/interface/cst.wit.jinja2 b/crates/codegen/runtime/cargo/wasm/src/runtime/interface/cst.wit.jinja2
@@ -11,7 +11,14 @@ interface cst {
             stub3,
         {%- else %}
             {%- for variant in model.kinds.nonterminal_kinds %}
-                {{ variant | wit_case }},
+                /// This kind represents a `{{ variant.id }}` node, with the following structure: 
+                ///
+                /// ```ebnf
+                {%- for line in variant.documentation | split(pat="\n") %}
+                /// {{ line }}
+                {%- endfor %}
+                /// ```
+                {{ variant.id | wit_case }},
             {%- endfor %}
         {%- endif %}
     }
@@ -35,7 +42,14 @@ interface cst {
             stub3,
         {%- else %}
             {%- for variant in model.kinds.terminal_kinds %}
-                {{ variant | wit_case }},
+                /// This kind represents a `{{ variant.id }}` node, with the following structure: 
+                ///
+                /// ```ebnf
+                {%- for line in variant.documentation | split(pat="\n") %}
+                /// {{ line }}
+                {%- endfor %}
+                /// ```
+                {{ variant.id | wit_case }},
             {%- endfor %}
         {%- endif %}
     }

diff --git a/crates/codegen/runtime/generator/Cargo.toml b/crates/codegen/runtime/generator/Cargo.toml
@@ -7,6 +7,7 @@ publish = false
 
 [dependencies]
 anyhow = { workspace = true }
+codegen_ebnf = { workspace = true }
 codegen_language_definition = { workspace = true }
 indexmap = { workspace = true }
 Inflector = { workspace = true }

diff --git a/crates/codegen/runtime/generator/src/kinds/mod.rs b/crates/codegen/runtime/generator/src/kinds/mod.rs
@@ -1,15 +1,51 @@
 use std::collections::BTreeSet;
+use std::rc::Rc;
 
+use codegen_ebnf::{EbnfModel, PlainWriter};
 use codegen_language_definition::model::{self, Identifier, Item, PredefinedLabel};
 use serde::Serialize;
 use strum::VariantNames;
 
+#[derive(Serialize, Ord, Eq, PartialEq, PartialOrd)]
+pub struct Kind {
+    id: Identifier,
+    documentation: String,
+}
+
+struct EbnfBuilder {
+    model: EbnfModel,
+    writer: PlainWriter,
+}
+
+impl EbnfBuilder {
+    fn new(language: &Rc<model::Language>) -> EbnfBuilder {
+        EbnfBuilder {
+            model: EbnfModel::build(language),
+            writer: PlainWriter::default(),
+        }
+    }
+
+    fn build(&mut self, name: &Identifier) -> Kind {
+        if self.model.serialize(name, &mut self.writer).is_ok() {
+            Kind {
+                id: name.clone(),
+                documentation: self.writer.flush(),
+            }
+        } else {
+            Kind {
+                id: name.clone(),
+                documentation: String::new(),
+            }
+        }
+    }
+}
+
 #[derive(Serialize)]
 pub struct KindsModel {
     /// Defines the `NonterminalKind` enum variants.
-    nonterminal_kinds: BTreeSet<Identifier>,
+    nonterminal_kinds: BTreeSet<Kind>,
     /// Defines the `TerminalKind` enum variants.
-    terminal_kinds: BTreeSet<Identifier>,
+    terminal_kinds: BTreeSet<Kind>,
     /// Defines `TerminalKind::is_trivia` method.
     trivia_scanner_names: BTreeSet<Identifier>,
     /// Defines `EdgeLabel` enum variants.
@@ -37,32 +73,34 @@ impl Default for KindsModel {
 }
 
 impl KindsModel {
-    pub fn from_language(language: &model::Language) -> Self {
+    pub fn from_language(language: &Rc<model::Language>) -> Self {
+        let mut kind_builder = EbnfBuilder::new(language);
+
         let terminal_kinds = language
             .items()
             .filter(|item| item.is_terminal() && !matches!(item, Item::Fragment { .. }))
-            .map(|item| item.name().clone())
+            .map(|item| kind_builder.build(item.name()))
             .collect();
 
         let mut nonterminal_kinds = BTreeSet::default();
         for item in language.items() {
             match item {
                 Item::Struct { item } => {
-                    nonterminal_kinds.insert(item.name.clone());
+                    nonterminal_kinds.insert(kind_builder.build(&item.name));
                 }
                 Item::Enum { item } => {
-                    nonterminal_kinds.insert(item.name.clone());
+                    nonterminal_kinds.insert(kind_builder.build(&item.name));
                 }
                 Item::Repeated { item } => {
-                    nonterminal_kinds.insert(item.name.clone());
+                    nonterminal_kinds.insert(kind_builder.build(&item.name));
                 }
                 Item::Separated { item } => {
-                    nonterminal_kinds.insert(item.name.clone());
+                    nonterminal_kinds.insert(kind_builder.build(&item.name));
                 }
                 Item::Precedence { item } => {
-                    nonterminal_kinds.insert(item.name.clone());
+                    nonterminal_kinds.insert(kind_builder.build(&item.name));
                     for op in &item.precedence_expressions {
-                        nonterminal_kinds.insert(op.name.clone());
+                        nonterminal_kinds.insert(kind_builder.build(&op.name));
                     }
                 }
                 // Terminals