Hide Lexer-related facilities from the public API (#728)

Ticks a box in #640

.changeset/green-pets-invent.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+Remove Language#scan API; use the parser API instead

crates/codegen/parser/runtime/src/kinds.rs

@@ -16,7 +16,7 @@ use napi_derive::napi;
 #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))]
 pub enum TokenKind {
     SKIPPED,
-    XXX,
+    // Expanded by the template engine
 }
 
 #[derive(
@@ -35,7 +35,7 @@ pub enum TokenKind {
 pub enum RuleKind {
     LeadingTrivia,
     TrailingTrivia,
-    XXX,
+    // Expanded by the template engine
 }
 
 impl RuleKind {
@@ -71,18 +71,16 @@ pub enum FieldName {
 }
 
 /// The lexical context of the scanner.
-#[derive(strum_macros::FromRepr)]
-#[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "language"))]
-#[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))]
-pub enum LexicalContext {
-    XXX,
+#[derive(strum_macros::FromRepr, Clone, Copy)]
+pub(crate) enum LexicalContext {
+    // Expanded by the template engine
 }
 
 /// Marker trait for type-level [`LexicalContext`] variants.
-pub trait IsLexicalContext {
+pub(crate) trait IsLexicalContext {
     /// Returns a run-time [`LexicalContext`] value.
     fn value() -> LexicalContext;
 }
 
 #[allow(non_snake_case)]
-pub mod LexicalContextType {}
+pub(crate) mod LexicalContextType {}

crates/codegen/parser/runtime/src/lexer.rs

@@ -2,7 +2,7 @@ use crate::cst::{self, NamedNode};
 use crate::kinds::{IsLexicalContext, TokenKind};
 use crate::support::{ParserContext, ParserResult};
 
-pub trait Lexer {
+pub(crate) trait Lexer {
     // Generated by the templating engine
     #[doc(hidden)]
     fn next_token<LexCtx: IsLexicalContext>(

crates/codegen/parser/runtime/src/support/mod.rs

@@ -18,7 +18,8 @@ pub use optional_helper::OptionalHelper;
 pub use parser_function::ParserFunction;
 pub use parser_result::ParserResult;
 pub use precedence_helper::PrecedenceHelper;
-pub use recovery::RecoverFromNoMatch;
+#[allow(unused_imports)] // Used when copied to shipped crate
+pub(crate) use recovery::RecoverFromNoMatch;
 pub use repetition_helper::{OneOrMoreHelper, ZeroOrMoreHelper};
 pub use separated_helper::SeparatedHelper;
 pub use sequence_helper::SequenceHelper;

crates/codegen/parser/runtime/src/support/recovery.rs

@@ -9,7 +9,7 @@ use crate::text_index::{TextRange, TextRangeExtensions as _};
 
 /// An explicit parameter for the [`ParserResult::recover_until_with_nested_delims`] method.
 #[derive(Clone, Copy)]
-pub enum RecoverFromNoMatch {
+pub(crate) enum RecoverFromNoMatch {
     Yes,
     No,
 }
@@ -41,7 +41,7 @@ impl ParserResult {
     /// Respects nested delimiters, i.e. the `expected` token is only accepted if it's not nested inside.
     /// Does not consume the `expected` token.
     #[must_use]
-    pub fn recover_until_with_nested_delims<L: Lexer, LexCtx: IsLexicalContext>(
+    pub(crate) fn recover_until_with_nested_delims<L: Lexer, LexCtx: IsLexicalContext>(
         self,
         input: &mut ParserContext<'_>,
         lexer: &L,
@@ -117,7 +117,7 @@ impl ParserResult {
 /// Does not consume the `expected` token.
 ///
 /// Returns the found token and the range of skipped tokens on success.
-pub fn skip_until_with_nested_delims<L: Lexer, LexCtx: IsLexicalContext>(
+pub(crate) fn skip_until_with_nested_delims<L: Lexer, LexCtx: IsLexicalContext>(
     input: &mut ParserContext<'_>,
     lexer: &L,
     until: TokenKind,

crates/codegen/parser/runtime/src/support/separated_helper.rs

@@ -10,7 +10,7 @@ use crate::text_index::TextRangeExtensions;
 pub struct SeparatedHelper;
 
 impl SeparatedHelper {
-    pub fn run<L: Lexer, LexCtx: IsLexicalContext>(
+    pub(crate) fn run<L: Lexer, LexCtx: IsLexicalContext>(
         input: &mut ParserContext<'_>,
         lexer: &L,
         body_parser: impl Fn(&mut ParserContext<'_>) -> ParserResult,

crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2

@@ -83,24 +83,22 @@ pub enum TokenKind {
     {%- endfor -%}
 }
 
-#[derive(strum_macros::FromRepr)]
 /// The lexical context of the scanner.
-#[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "language"))]
-#[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))]
-pub enum LexicalContext {
+#[derive(strum_macros::FromRepr, Clone, Copy)]
+pub(crate) enum LexicalContext {
     {%- for context in code.scanner_contexts %}
         {{ context.name }},
     {%- endfor %}
 }
 
 /// Marker trait for type-level [`LexicalContext`] variants.
-pub trait IsLexicalContext {
+pub(crate) trait IsLexicalContext {
     /// Returns a run-time [`LexicalContext`] value.
     fn value() -> LexicalContext;
 }
 
 #[allow(non_snake_case)]
-pub mod LexicalContextType {
+pub(crate) mod LexicalContextType {
     use super::{IsLexicalContext, LexicalContext};
 
     {%- for context in code.scanner_contexts %}

crates/codegen/parser/runtime/src/templates/language.rs.jinja2

@@ -12,8 +12,9 @@ use semver::Version;
 use napi_derive::napi;
 
 use crate::cst;
-pub use crate::kinds::LexicalContext;
-use crate::kinds::{FieldName, IsLexicalContext, LexicalContextType, RuleKind, TokenKind};
+use crate::kinds::{
+    FieldName, IsLexicalContext, LexicalContext, LexicalContextType, RuleKind, TokenKind,
+};
 use crate::lexer::Lexer;
 #[cfg(feature = "slang_napi_interfaces")]
 use crate::napi::napi_parse_output::ParseOutput as NAPIParseOutput;
@@ -91,16 +92,6 @@ impl Language {
         fn {{ function.0 | snake_case }}(&self, input: &mut ParserContext<'_>) -> bool { {{ function.1 }} }
     {% endfor %}
 
-    pub fn scan(&self, lexical_context: LexicalContext, input: &str) -> Option<TokenKind> {
-        let mut input = ParserContext::new(input);
-        match lexical_context {
-            {%- for lexical_context in code.scanner_contexts -%}
-                LexicalContext::{{ lexical_context.name }} =>
-                    Lexer::next_token::<LexicalContextType::{{ lexical_context.name }}>(self, &mut input),
-            {%- endfor -%}
-        }
-    }
-
     pub fn parse(&self, kind: RuleKind, input: &str) -> ParseOutput {
         match kind {
             {%- for function in code.parser_functions -%}
@@ -214,11 +205,6 @@ impl Language {
         return Self::SUPPORTED_VERSIONS.iter().map(|v| v.to_string()).collect();
     }
 
-    #[napi(js_name = "scan", ts_return_type = "kinds.TokenKind | null", catch_unwind)]
-    pub fn scan_napi(&self, lexical_context: LexicalContext, input: String) -> Option<TokenKind> {
-        self.scan(lexical_context, input.as_str())
-    }
-
     #[napi(js_name = "parse", ts_return_type = "parse_output.ParseOutput", catch_unwind)]
     pub fn parse_napi(
         &self,

crates/solidity/outputs/cargo/tests/src/cst_output/runner.rs

@@ -6,7 +6,7 @@ use infra_utils::codegen::Codegen;
 use infra_utils::paths::PathExtensions;
 use slang_solidity::kinds::RuleKind;
 use slang_solidity::language::Language;
-use solidity_testing_utils::cst_snapshots::CstSnapshots;
+use solidity_testing_utils::cst_snapshots::{CstSnapshots, WithWhitespace};
 use strum_macros::Display;
 
 use crate::cst_output::generated::VERSION_BREAKS;
@@ -34,10 +34,10 @@ pub fn run(parser_name: &str, test_name: &str) -> Result<()> {
     let mut last_output = None;
 
     for version in VERSION_BREAKS {
-        let rule_kind = RuleKind::from_str(parser_name)
+        let tested_rule_kind = RuleKind::from_str(parser_name)
             .unwrap_or_else(|_| panic!("No such parser: {parser_name}"));
 
-        let output = Language::new(version.clone())?.parse(rule_kind, &source);
+        let output = Language::new(version.clone())?.parse(tested_rule_kind, &source);
 
         let output = match last_output {
             // Skip this version if it produces the same output.
@@ -60,7 +60,13 @@ pub fn run(parser_name: &str, test_name: &str) -> Result<()> {
             TestStatus::Failure
         };
 
-        let snapshot = CstSnapshots::render(&source, &errors, cursor)?;
+        let render_whitespace = if tested_rule_kind.is_trivia() {
+            WithWhitespace::Yes
+        } else {
+            WithWhitespace::No
+        };
+
+        let snapshot = CstSnapshots::render(&source, &errors, cursor, render_whitespace)?;
 
         let snapshot_path = test_dir
             .join("generated")

crates/solidity/outputs/cargo/tests/src/lib.rs

@@ -3,5 +3,4 @@
 mod cst_output;
 mod doc_examples;
 mod errors;
-mod scanner;
 mod versions;