Use DSL v2 instead of v0 when collecting breaking versions for tests (#…

…684) Part of #637 #638 The other use case is in `codegen/spec/src/grammar.rs` but that the rest of the code uses v0 for spec-generation, so I didn't change that not to mix definitions and not to step on your toes @OmarTawfik for #637. --------- Co-authored-by: Omar Tawfik <[email protected]>
NomicFoundation · Dec 4, 2023 · 0b3a2af · 0b3a2af
1 parent 3619746
commit 0b3a2af
Show file tree

Hide file tree

Showing 14 changed files with 157 additions and 73 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/codegen/language/definition/src/model/manifest.rs b/crates/codegen/language/definition/src/model/manifest.rs
@@ -1,9 +1,9 @@
-use crate::model::{Identifier, Item, TriviaParser};
+use crate::model::{Field, Identifier, Item, TriviaParser, VersionSpecifier};
 use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens};
 use indexmap::IndexSet;
 use semver::Version;
 use serde::{Deserialize, Serialize};
-use std::rc::Rc;
+use std::{collections::BTreeSet, rc::Rc};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
@@ -36,3 +36,103 @@ pub struct Topic {
 
     pub items: Vec<Rc<Item>>,
 }
+
+impl Language {
+    /// Returns every item in the language definition.
+    pub fn items(&self) -> impl Iterator<Item = &Item> {
+        self.sections
+            .iter()
+            .flat_map(|section| &*section.topics)
+            .flat_map(|topic| topic.items.iter().map(AsRef::as_ref))
+    }
+
+    /// Returns a flattened iterator over items along with section and topic they belong to.
+    pub fn items_with_section(&self) -> impl Iterator<Item = (&Section, &Topic, &Rc<Item>)> {
+        self.sections.iter().flat_map(|section| {
+            section
+                .topics
+                .iter()
+                .flat_map(move |topic| topic.items.iter().map(move |item| (section, topic, item)))
+        })
+    }
+
+    /// Collects all versions that change the language grammar in a breaking way.
+    pub fn collect_breaking_versions(&self) -> BTreeSet<Version> {
+        let first = self.versions.first().unwrap().clone();
+        let mut res = BTreeSet::from_iter([first]);
+
+        let mut add_spec = |spec: &Option<VersionSpecifier>| {
+            let Some(spec) = spec else {
+                return;
+            };
+
+            match spec.clone() {
+                VersionSpecifier::Never => (),
+                VersionSpecifier::From { from } => {
+                    res.insert(from);
+                }
+                VersionSpecifier::Till { till } => {
+                    res.insert(till);
+                }
+                VersionSpecifier::Range { from, till } => {
+                    res.insert(from);
+                    res.insert(till);
+                }
+            }
+        };
+
+        for item in self.items() {
+            match item {
+                Item::Struct { item } => {
+                    add_spec(&item.enabled);
+                    for field in item.fields.values() {
+                        match field {
+                            Field::Required { .. } => (),
+                            Field::Optional { enabled, .. } => add_spec(enabled),
+                        }
+                    }
+                }
+                Item::Enum { item } => {
+                    add_spec(&item.enabled);
+                    for variant in &item.variants {
+                        add_spec(&variant.enabled);
+                    }
+                }
+                Item::Repeated { item } => add_spec(&item.enabled),
+                Item::Separated { item } => add_spec(&item.enabled),
+                Item::Precedence { item } => {
+                    add_spec(&item.enabled);
+                    for prec in &item.precedence_expressions {
+                        for op in &prec.operators {
+                            add_spec(&op.enabled);
+                            for field in op.fields.values() {
+                                match field {
+                                    Field::Required { .. } => (),
+                                    Field::Optional { enabled, .. } => add_spec(enabled),
+                                }
+                            }
+                        }
+                    }
+                    for prim in &item.primary_expressions {
+                        add_spec(&prim.enabled);
+                    }
+                }
+                Item::Keyword { item } => {
+                    for definition in &item.definitions {
+                        add_spec(&definition.enabled);
+                        add_spec(&definition.reserved);
+                    }
+                }
+                Item::Token { item } => {
+                    for definition in &item.definitions {
+                        add_spec(&definition.enabled);
+                    }
+                }
+                Item::Fragment { item } => add_spec(&item.enabled),
+                Item::Trivia { .. } => {}
+            }
+        }
+
+        res
+    }
+}
diff --git a/crates/codegen/testing/Cargo.toml b/crates/codegen/testing/Cargo.toml
@@ -7,5 +7,5 @@ publish = false
 
 [dependencies]
 anyhow = { workspace = true }
-codegen_schema = { workspace = true }
+codegen_language_definition = { workspace = true }
 infra_utils = { workspace = true }
diff --git a/crates/codegen/testing/src/cst_output.rs b/crates/codegen/testing/src/cst_output.rs
@@ -4,14 +4,14 @@ use std::{
 };
 
 use anyhow::{bail, Result};
-use codegen_schema::types::LanguageDefinition;
+use codegen_language_definition::model::Language;
 use infra_utils::{
     codegen::{Codegen, CodegenReadWrite},
     paths::{FileWalker, PathExtensions},
 };
 
 pub fn generate_cst_output_tests(
-    language: &LanguageDefinition,
+    language: &Language,
     data_dir: &Path,
     output_dir: &Path,
 ) -> Result<()> {
@@ -76,7 +76,7 @@ fn collect_parser_tests(data_dir: &Path) -> Result<BTreeMap<String, BTreeSet<Str
 }
 
 fn generate_mod_file(
-    language: &LanguageDefinition,
+    language: &Language,
     codegen: &mut CodegenReadWrite,
     mod_file_path: &Path,
     parser_tests: &BTreeMap<String, BTreeSet<String>>,
@@ -86,7 +86,7 @@ fn generate_mod_file(
         .map(|parser_name| format!("#[allow(non_snake_case)] mod {parser_name};"))
         .collect::<String>();
 
-    let version_breaks = language.collect_version_breaks();
+    let version_breaks = language.collect_breaking_versions();
     let version_breaks_len = version_breaks.len();
     let version_breaks_str = version_breaks
         .iter()

diff --git a/crates/codegen/testing/src/lib.rs b/crates/codegen/testing/src/lib.rs
@@ -3,15 +3,15 @@ mod cst_output;
 use std::path::Path;
 
 use anyhow::Result;
-use codegen_schema::types::LanguageDefinition;
+use codegen_language_definition::model::Language;
 
 use crate::cst_output::generate_cst_output_tests;
 
 pub trait TestingGeneratorExtensions {
     fn generate_cst_output_tests(&self, snapshots_dir: &Path, output_dir: &Path) -> Result<()>;
 }
 
-impl TestingGeneratorExtensions for LanguageDefinition {
+impl TestingGeneratorExtensions for Language {
     fn generate_cst_output_tests(&self, data_dir: &Path, output_dir: &Path) -> Result<()> {
         generate_cst_output_tests(self, data_dir, output_dir)
     }

diff --git a/crates/solidity/inputs/language/src/grammar.rs b/crates/solidity/inputs/language/src/grammar.rs
@@ -23,28 +23,20 @@ use codegen_language_definition::model::Identifier;
 use codegen_language_definition::model::Item;
 use indexmap::IndexMap;
 
-use crate::definition::SolidityDefinition;
-
-/// Materializes the DSL v2 model ([`SolidityDefinition`]) into [`Grammar`].
+/// Materializes the DSL v2 model ([`model::Language`]) into [`Grammar`].
 pub trait GrammarConstructorDslV2 {
-    fn from_dsl_v2() -> Rc<Grammar>;
+    fn from_dsl_v2(lang: &model::Language) -> Grammar;
 }
 
 impl GrammarConstructorDslV2 for Grammar {
-    fn from_dsl_v2() -> Rc<Grammar> {
-        let lang = SolidityDefinition::create();
-
-        let mut items = HashMap::new();
-
-        for section in lang.sections {
-            for topic in section.topics {
-                let ctx = topic.lexical_context;
-
-                for item in topic.items {
-                    items.insert(item.name().clone(), (ctx.clone(), Rc::clone(&item)));
-                }
-            }
-        }
+    fn from_dsl_v2(lang: &model::Language) -> Grammar {
+        // Collect language items into a lookup table to speed up resolution
+        let mut items = HashMap::from_iter(lang.items_with_section().map(|(_, topic, item)| {
+            (
+                item.name().clone(),
+                (topic.lexical_context.clone(), Rc::clone(item)),
+            )
+        }));
 
         // TODO(#638): To minimize regression in the parser migration, we keep the existing DSL v1 model
         // of SourceUnit being followed by `EndOfFileTrivia`.
@@ -95,12 +87,12 @@ impl GrammarConstructorDslV2 for Grammar {
 
         let trailing_trivia = Rc::new(NamedTriviaParser {
             name: "TrailingTrivia",
-            def: resolve_trivia(lang.trailing_trivia, &mut ctx),
+            def: resolve_trivia(lang.trailing_trivia.clone(), &mut ctx),
         }) as Rc<dyn TriviaParserDefinition>;
 
         let eof_trivia = Rc::new(NamedTriviaParser {
             name: "EndOfFileTrivia",
-            def: resolve_trivia(lang.leading_trivia, &mut ctx),
+            def: resolve_trivia(lang.leading_trivia.clone(), &mut ctx),
         }) as Rc<dyn TriviaParserDefinition>;
 
         ctx.resolved.insert(
@@ -159,9 +151,9 @@ impl GrammarConstructorDslV2 for Grammar {
             .iter()
             .map(|(name, elem)| (name.to_string().leak() as &_, elem.clone()));
 
-        Rc::new(Grammar {
+        Grammar {
             name: lang.name.to_string(),
-            versions: BTreeSet::from_iter(lang.versions),
+            versions: BTreeSet::from_iter(lang.versions.clone()),
             leading_trivia_parser: leading_trivia.clone(),
             trailing_trivia_parser: trailing_trivia.clone(),
             elements: HashMap::from_iter(
@@ -171,7 +163,7 @@ impl GrammarConstructorDslV2 for Grammar {
                         .map(|elem| (elem.name(), elem.into())),
                 ),
             ),
-        })
+        }
     }
 }
 

diff --git a/crates/solidity/outputs/cargo/build/src/main.rs b/crates/solidity/outputs/cargo/build/src/main.rs
@@ -4,7 +4,7 @@ use cargo_emit::rerun_if_changed;
 use codegen_grammar::Grammar;
 use codegen_parser_generator::code_generator::CodeGenerator;
 use infra_utils::{cargo::CargoWorkspace, paths::PathExtensions};
-use solidity_language::GrammarConstructorDslV2;
+use solidity_language::{GrammarConstructorDslV2, SolidityDefinition};
 
 // Instead of the soure crate calling codegen APIs directly in the build script, it invokes this binary, which in turn
 // calls the codegen APIs (and hence why it's emitting `cargo:` directives).
@@ -17,7 +17,8 @@ use solidity_language::GrammarConstructorDslV2;
 fn main() -> Result<()> {
     // Generate files in the source crate:
     {
-        let grammar = Grammar::from_dsl_v2();
+        let lang_def = SolidityDefinition::create();
+        let grammar = Grammar::from_dsl_v2(&lang_def);
 
         let crate_dir = CargoWorkspace::locate_source_crate("slang_solidity")?;
 

diff --git a/crates/solidity/outputs/cargo/tests/Cargo.toml b/crates/solidity/outputs/cargo/tests/Cargo.toml
@@ -7,7 +7,6 @@ publish = false
 
 [build-dependencies]
 anyhow = { workspace = true }
-codegen_schema = { workspace = true }
 codegen_testing = { workspace = true }
 infra_utils = { workspace = true }
 solidity_language = { workspace = true }

diff --git a/crates/solidity/outputs/cargo/tests/build.rs b/crates/solidity/outputs/cargo/tests/build.rs
@@ -1,17 +1,14 @@
 use anyhow::Result;
-use codegen_schema::types::LanguageDefinition;
 use codegen_testing::TestingGeneratorExtensions;
 use infra_utils::cargo::CargoWorkspace;
-use solidity_language::SolidityLanguageExtensions;
+use solidity_language::SolidityDefinition;
 
 fn main() -> Result<()> {
-    let language = LanguageDefinition::load_solidity()?;
+    let lang_def = SolidityDefinition::create();
 
-    language.generate_cst_output_tests(
+    lang_def.generate_cst_output_tests(
         &CargoWorkspace::locate_source_crate("solidity_testing_snapshots")?.join("cst_output"),
         &CargoWorkspace::locate_source_crate("solidity_cargo_tests")?
             .join("src/cst_output/generated"),
-    )?;
-
-    Ok(())
+    )
 }
diff --git a/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs b/crates/solidity/outputs/cargo/tests/src/cst_output/generated/mod.rs
diff --git a/crates/solidity/outputs/npm/build/src/main.rs b/crates/solidity/outputs/npm/build/src/main.rs
@@ -4,7 +4,7 @@ use cargo_emit::rerun_if_changed;
 use codegen_grammar::Grammar;
 use codegen_parser_generator::code_generator::CodeGenerator;
 use infra_utils::{cargo::CargoWorkspace, paths::PathExtensions};
-use solidity_language::GrammarConstructorDslV2;
+use solidity_language::{GrammarConstructorDslV2, SolidityDefinition};
 
 // Instead of the soure crate calling codegen APIs directly in the build script, it invokes this binary, which in turn
 // calls the codegen APIs (and hence why it's emitting `cargo:` directives).
@@ -17,7 +17,8 @@ use solidity_language::GrammarConstructorDslV2;
 fn main() -> Result<()> {
     // Generate files in the source crate:
     {
-        let grammar = Grammar::from_dsl_v2();
+        let lang_def = SolidityDefinition::create();
+        let grammar = Grammar::from_dsl_v2(&lang_def);
 
         let crate_dir = CargoWorkspace::locate_source_crate("solidity_npm_crate")?;
 

diff --git a/crates/solidity/testing/sanctuary/Cargo.toml b/crates/solidity/testing/sanctuary/Cargo.toml
@@ -7,7 +7,6 @@ publish = false
 
 [dependencies]
 anyhow = { workspace = true }
-codegen_schema = { workspace = true }
 infra_utils = { workspace = true }
 indicatif = { workspace = true }
 rayon = { workspace = true }

diff --git a/crates/solidity/testing/sanctuary/src/main.rs b/crates/solidity/testing/sanctuary/src/main.rs
@@ -4,12 +4,11 @@ mod reporting;
 use std::{collections::BTreeSet, path::Path};
 
 use anyhow::Result;
-use codegen_schema::types::LanguageDefinition;
 use infra_utils::paths::PathExtensions;
 use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
 use semver::Version;
 use slang_solidity::{kinds::ProductionKind, language::Language};
-use solidity_language::SolidityLanguageExtensions;
+use solidity_language::SolidityDefinition;
 use solidity_testing_utils::version_pragmas::extract_version_pragmas;
 
 use crate::{
@@ -18,11 +17,10 @@ use crate::{
 };
 
 fn main() {
+    let versions = SolidityDefinition::create().collect_breaking_versions();
+
     // Fail the parent process if a child thread panics:
     std::panic::catch_unwind(|| -> Result<()> {
-        let language = &LanguageDefinition::load_solidity()?;
-        let versions = language.collect_version_breaks();
-
         for dataset in get_all_datasets()? {
             process_dataset(&dataset, &versions)?;
         }