refactor: Use collect_breaking_versions from v2 directly in PG (Nomic…

…Foundation#1002)

Part of NomicFoundation#638

Follow-up to NomicFoundation#991

Pretty straightforward: instead of visiting the previously built v1
definition structure, we defer to `Language::collect_breaking_changes`
as the definitions overlap - the breaking changes are defined as
versions in which the syntax items may be evaluated differently, which
means that these are exactly the versions that will be referenced for
the conditional syntax item evaluation in the parser/lexer.

Refactor `BuiltInLabel` to avoid duplication (NomicFoundation#992)

Spin off of NomicFoundation#976

Moves the `BuiltInLabel` enum from the parser generator into the
language definition and remove duplication in the `kinds` template.

add wit

generating wit and glue

stub adaptors, wit feature flag

glue macros

remove wit_bindgen

fix wit gen paths

add wit-bindgen

export the kinds

pub export macro for wit

improve export macro

world => slang

fully implement glue

convert query matches

refactor ffi glue macros

refactor wit

variant rather than enum

back to enum

.vscode/settings.json

@@ -3,7 +3,8 @@
   "editor.rulers": [120],
   "files.associations": {
     "**/documentation/overrides/**/*.html": "jinja-html",
-    "*.ts.jinja2": "jinja-js" // until this is merged: https://github.com/samuelcolvin/jinjahtml-vscode/pull/148
+    "*.ts.jinja2": "jinja-js", // until this is merged: https://github.com/samuelcolvin/jinjahtml-vscode/pull/148
+    "*.wit.jinja2": "jinja" // until this is merged: https://github.com/samuelcolvin/jinjahtml-vscode/pull/148
   },
   "editor.unicodeHighlight.allowedCharacters": {
     "꞉": true // used in CST snapshot tests

Cargo.toml

@@ -117,6 +117,7 @@ napi-derive = { version = "2.16.10" }
 nom = { version = "7.1.3" }
 num-format = { version = "0.4.4" }
 once_cell = { version = "1.19.0" }
+paste = { version = "1.0.15" }
 proc-macro2 = { version = "1.0.86" }
 quote = { version = "1.0.36" }
 rayon = { version = "1.10.0" }
@@ -148,6 +149,8 @@ thiserror = { version = "1.0.63" }
 toml = { version = "0.8.19" }
 trybuild = { version = "1.0.99" }
 url = { version = "2.4.1", features = ["serde"] }
+wit-bindgen = { version = "0.26.0" }
+wit-bindgen-cli = { version = "0.26.0" }
 
 [workspace.lints.rust]
 unused_crate_dependencies = "warn"

crates/codegen/runtime/cargo/Cargo.toml

@@ -10,19 +10,22 @@ description = "Cargo runtime copied over by codegen"
 # __RUST_PRODUCT_CRATE_FEATURES__ (keep in sync)
 [features]
 default = ["cli"]
+wit = ["dep:paste"]
 cli = ["dep:ariadne", "dep:clap", "dep:serde_json"]
 __experimental_bindings_api = ["dep:metaslang_bindings"]
 __private_testing_utils = ["dep:ariadne"]
 
 [build-dependencies]
 anyhow = { workspace = true }
 codegen_runtime_generator = { workspace = true }
+infra_utils = { workspace = true }
 
 [dependencies]
 ariadne = { workspace = true, optional = true }
 clap = { workspace = true, optional = true }
 metaslang_bindings = { workspace = true, optional = true }
 metaslang_cst = { workspace = true }
+paste = { workspace = true, optional = true }
 semver = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true, optional = true }

crates/codegen/runtime/cargo/build.rs

@@ -1,6 +1,11 @@
 use anyhow::Result;
 use codegen_runtime_generator::OutputLanguage;
+use infra_utils::cargo::CargoWorkspace;
 
 fn main() -> Result<()> {
-    OutputLanguage::Cargo.generate_stubs()
+    OutputLanguage::Cargo.generate_stubs()?;
+    let output_dir =
+        CargoWorkspace::locate_source_crate("codegen_runtime_cargo")?.join("src/runtime");
+    OutputLanguage::Cargo.wit_bindgen(&output_dir)?;
+    Ok(())
 }

crates/codegen/runtime/cargo/src/runtime/kinds/mod.rs.jinja2

@@ -1,6 +1,8 @@
 #[cfg(feature = "__private_napi_interfaces")]
 use napi_derive::napi;
 
+// This needs to stay in sync with the wit-bindgen output
+{% if model.kinds.nonterminal_kinds|length <= 256 %} #[repr(u8)] {% else %} #[repr(u16)] {% endif %}
 #[derive(
     Debug,
     Eq,
@@ -31,6 +33,8 @@ pub enum NonterminalKind {
 
 impl metaslang_cst::NonterminalKind for NonterminalKind {}
 
+// This needs to stay in sync with the wit-bindgen output
+{% if model.kinds.labels|length <= 256 %} #[repr(u8)] {% else %} #[repr(u16)] {% endif %}
 #[derive(
     Debug,
     Eq,
@@ -67,6 +71,8 @@ pub enum EdgeLabel {
 
 impl metaslang_cst::EdgeLabel for EdgeLabel {}
 
+// This needs to stay in sync with the wit-bindgen output
+{% if model.kinds.terminal_kinds|length <= 256 %} #[repr(u8)] {% else %} #[repr(u16)] {% endif %}
 #[derive(
     Debug,
     Eq,
@@ -120,6 +126,8 @@ impl metaslang_cst::TerminalKind for TerminalKind {
 }
 
 /// The lexical context of the scanner.
+// This needs to stay in sync with the wit-bindgen output
+{% if model.kinds.lexical_contexts|length <= 256 %} #[repr(u8)] {% else %} #[repr(u16)] {% endif %}
 #[derive(strum_macros::FromRepr, Clone, Copy)]
 pub(crate) enum LexicalContext {
     {%- if rendering_in_stubs -%}

crates/codegen/runtime/cargo/src/runtime/mod.rs

@@ -8,6 +8,9 @@ pub mod parse_output;
 pub mod query;
 pub mod text_index;
 
+#[cfg(feature = "wit")]
+pub mod wit;
+
 #[cfg(feature = "__private_napi_interfaces")]
 pub mod napi_interface;
 

crates/codegen/runtime/cargo/src/runtime/query/mod.rs

@@ -3,5 +3,6 @@ use metaslang_cst::query;
 use crate::cst::KindTypes;
 
 pub type Query = query::Query<KindTypes>;
+pub type QueryError = query::QueryError;
 pub type QueryMatch = query::QueryMatch<KindTypes>;
 pub type QueryMatchIterator = query::QueryMatchIterator<KindTypes>;

crates/codegen/runtime/cargo/src/runtime/wit/cst.rs

@@ -0,0 +1,65 @@
+use super::{define_rc_wrapper, ffi, rust, FromFFI, IntoFFI};
+
+//================================================
+//
+// resource nonterminal-node
+//
+//================================================
+
+define_rc_wrapper! { NonterminalNode {
+    fn kind(&self) -> ffi::NonterminalKind {
+        self._borrow_ffi().kind._into_ffi()
+    }
+
+    fn text_len(&self) -> ffi::TextIndex {
+        self._borrow_ffi().text_len._into_ffi()
+    }
+
+    fn children(&self) -> Vec<ffi::Node> {
+        todo!()
+    }
+
+    fn create_cursor(&self, text_offset: ffi::TextIndex) -> ffi::Cursor {
+        std::rc::Rc::clone(self._borrow_ffi()).cursor_with_offset(text_offset._from_ffi())._into_ffi()
+    }
+
+    fn unparse(&self) -> String {
+        std::rc::Rc::clone(self._borrow_ffi()).unparse()
+    }
+} }
+
+//================================================
+//
+// resource terminal-node
+//
+//================================================
+
+define_rc_wrapper! { TerminalNode {
+    fn kind(&self) -> ffi::TerminalKind {
+        self._borrow_ffi().kind._into_ffi()
+    }
+
+    fn text(&self) -> String {
+        self._borrow_ffi().text.clone()
+    }
+
+    fn text_len(&self) -> ffi::TextIndex {
+        rust::TextIndex::from(&self._borrow_ffi().text)._into_ffi()
+    }
+} }
+
+//================================================
+//
+// variant node
+//
+//================================================
+
+impl IntoFFI<ffi::Node> for rust::Node {
+    #[inline]
+    fn _into_ffi(self) -> ffi::Node {
+        match self {
+            Self::Nonterminal(node) => ffi::Node::Nonterminal(node._into_ffi()),
+            Self::Terminal(node) => ffi::Node::Terminal(node._into_ffi()),
+        }
+    }
+}

crates/codegen/runtime/cargo/src/runtime/wit/cursor.rs

@@ -0,0 +1,123 @@
+use super::{define_refcell_wrapper, ffi, rust, FromFFI, IntoFFI};
+//================================================
+//
+// resource cursor
+//
+//================================================
+
+define_refcell_wrapper! { Cursor {
+    fn reset(&self) {
+        self._borrow_mut_ffi().reset();
+    }
+
+    fn complete(&self) {
+        self._borrow_mut_ffi().complete();
+    }
+
+    fn is_completed(&self) -> bool {
+        self._borrow_ffi().is_completed()
+    }
+
+    fn clone(&self) -> ffi::Cursor {
+        self._borrow_ffi().clone()._into_ffi()
+    }
+
+    fn spawn(&self) -> ffi::Cursor {
+        self._borrow_ffi().spawn()._into_ffi()
+    }
+
+    fn node(&self) -> ffi::Node {
+        self._borrow_ffi().node()._into_ffi()
+    }
+
+    fn label(&self) -> Option<ffi::EdgeLabel> {
+        self._borrow_ffi().label().map(IntoFFI::_into_ffi)
+    }
+
+    fn text_offset(&self) -> ffi::TextIndex {
+        self._borrow_ffi().text_offset()._into_ffi()
+    }
+
+    fn text_range(&self) -> ffi::TextRange {
+        self._borrow_ffi().text_range()._into_ffi()
+    }
+
+    #[allow(clippy::cast_possible_truncation)]
+    fn depth(&self) -> u32 {
+        self._borrow_ffi().depth() as u32
+    }
+
+    fn ancestors(&self) -> Vec<ffi::NonterminalNode> {
+        self._borrow_ffi().ancestors().map(|x|x._into_ffi()).collect()
+    }
+
+    fn go_to_next(&self) -> bool {
+        self._borrow_mut_ffi().go_to_next()
+    }
+
+    fn go_to_next_non_descendent(&self) -> bool {
+        self._borrow_mut_ffi().go_to_next_non_descendent()
+    }
+
+    fn go_to_previous(&self) -> bool {
+        self._borrow_mut_ffi().go_to_previous()
+    }
+
+    fn go_to_parent(&self) -> bool {
+        self._borrow_mut_ffi().go_to_parent()
+    }
+
+    fn go_to_first_child(&self) -> bool {
+        self._borrow_mut_ffi().go_to_first_child()
+    }
+
+    fn go_to_last_child(&self) -> bool {
+        self._borrow_mut_ffi().go_to_last_child()
+    }
+
+    fn go_to_nth_child(&self, child_number: u32) -> bool {
+        self._borrow_mut_ffi().go_to_nth_child(child_number as usize)
+    }
+
+    fn go_to_next_sibling(&self) -> bool {
+        self._borrow_mut_ffi().go_to_next_sibling()
+    }
+
+    fn go_to_previous_sibling(&self) -> bool {
+        self._borrow_mut_ffi().go_to_previous_sibling()
+    }
+
+    fn go_to_next_terminal(&self) -> bool {
+        self._borrow_mut_ffi().go_to_next_terminal()
+    }
+
+    fn go_to_next_terminal_with_kind(&self, kind: ffi::TerminalKind) -> bool {
+        self._borrow_mut_ffi().go_to_next_terminal_with_kind(kind._from_ffi())
+    }
+
+    fn go_to_next_terminal_with_kinds(&self, kinds: Vec<ffi::TerminalKind>) -> bool {
+        let kinds = kinds.into_iter().map(FromFFI::_from_ffi).collect::<Vec<_>>();
+        self._borrow_mut_ffi().go_to_next_terminal_with_kinds(&kinds)
+    }
+
+    fn go_to_next_nonterminal(&self) -> bool {
+        self._borrow_mut_ffi().go_to_next_nonterminal()
+    }
+
+    fn go_to_next_nonterminal_with_kind(&self, kind: ffi::NonterminalKind) -> bool {
+        self._borrow_mut_ffi().go_to_next_nonterminal_with_kind(kind._from_ffi())
+    }
+
+    fn go_to_next_nonterminal_with_kinds(&self, kinds: Vec<ffi::NonterminalKind>) -> bool {
+        let kinds = kinds.into_iter().map(FromFFI::_from_ffi).collect::<Vec<_>>();
+        self._borrow_mut_ffi().go_to_next_nonterminal_with_kinds(&kinds)
+    }
+
+    fn query(&self, queries: Vec<ffi::QueryBorrow<'_>>) -> ffi::QueryMatchIterator {
+        let queries:Vec<rust::Query> = queries.into_iter().map(|q|{
+            q._borrow_ffi().clone()
+        }).collect();
+
+        self._borrow_ffi().clone().query(queries)._into_ffi()
+    }
+} }

crates/codegen/runtime/cargo/src/runtime/wit/diagnostic.rs

@@ -0,0 +1,9 @@
+use super::{enum_to_enum, ffi, rust};
+
+//================================================
+//
+// enum severity
+//
+//================================================
+
+enum_to_enum!(Severity);