diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9704cd1..54bdaea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,6 +108,22 @@ to push further changes to your commits. Our policy is to not accept PRs that only fix typos in the documentation and code. We appreciate your effort, but we encourage you to focus on bugs and features instead. +## Tips + +### Testing custom Cairo compiler changes + +Sometimes you may happen to work on a feature to the Cairo compiler, +and you would like to test how it works in CairoLS. + +We have a script that edits the `Cargo.toml` file to use a local checkout of the Cairo compiler. +To use this tool, run: + +```shell +cargo xtask set-cairo-version --path ../path/to/cairo +``` + +And then you can `cargo build` CairoLS with your custom Cairo compiler changes. + --- Thanks! ❤️ ❤️ ❤️ diff --git a/Cargo.toml b/Cargo.toml index f43d1e4..2aa26d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,23 +14,38 @@ members = ["xtask"] [features] testing = [] +# Managing dependencies on crates from starkware-libs/cairo repository: +# +# The Cairo compiler is made of a bunch of crates that inter-depend on each other and have +# synchronised versioning. +# It is very important to use a single revision of these crates in the entire Cairo toolchain, +# which consists of Cairo compiler, Scarb, CairoLS and other tools. +# The toolchain is eventually built by Scarb, which depends on everything other as regular crates. +# To ensure that all crates in the toolchain use the same revision of Cairo crates, we use a patch +# mechanism that Cargo provides. +# Because Cargo requires patches to change the crate source, we have an unspoken contract that +# all tools *always* depend on some crates.io versions of Cairo crates and Scarb uses +# [patch.crates.io] table to set final git revision for everything. +# +# To keep our Cargo.toml following this contract, always use `cargo xtask set-cairo-version` +# for manipulating these dependencies. [dependencies] anyhow = "1" -cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } -cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-compiler = "*" +cairo-lang-defs = "*" +cairo-lang-diagnostics = "*" +cairo-lang-doc = "*" +cairo-lang-filesystem = "*" +cairo-lang-formatter = "*" +cairo-lang-lowering = "*" cairo-lang-macro = "0.1.1" +cairo-lang-parser = "*" +cairo-lang-project = "*" +cairo-lang-semantic = "*" +cairo-lang-starknet = "*" +cairo-lang-syntax = "*" +cairo-lang-test-plugin = "*" +cairo-lang-utils = "*" convert_case = "0.6.0" crossbeam = "0.8.4" governor = { version = "0.7.0", default-features = false, features = ["std", "quanta"] } @@ -59,12 +74,46 @@ libc = "0.2.167" [dev-dependencies] assert_fs = "1.1" +cairo-lang-test-utils = { version = "*", features = ["testing"] } cairo-language-server = { path = ".", features = ["testing"] } -cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174", features = ["testing"] } pathdiff = "0.2" pretty_assertions = "1.4.0" test-log = "0.2.16" +# Here we specify real dependency specifications for Cairo crates *if* currently we want to use +# a particular unreleased commit (which is frequent mid-development). +# We list all Cairo crates that go into CairoLS's compilation unit even if LS itself does not depend +# on some of them directly. +# This ensures no duplicate instances of Cairo crates are pulled in by mistake. +[patch.crates-io] +cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-eq-solver = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-plugins = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-proc-macros = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra-ap-change = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra-gas = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-syntax-codegen = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } +cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" } # The profile used for CI in pull requests. # External dependencies are built with optimisation enabled, diff --git a/xtask/src/set_cairo_version.rs b/xtask/src/set_cairo_version.rs index ca097b5..9a6af55 100644 --- a/xtask/src/set_cairo_version.rs +++ b/xtask/src/set_cairo_version.rs @@ -1,3 +1,4 @@ +use std::mem; use std::path::PathBuf; use anyhow::Result; @@ -8,7 +9,7 @@ use xshell::{Shell, cmd}; use crate::set_version; -/// Update `cairo-lang-*` crates in a proper way. +/// Update `cairo-lang-*` crates properly. #[derive(Parser)] pub struct Args { #[command(flatten)] @@ -45,14 +46,18 @@ pub fn main(args: Args) -> Result<()> { let mut cargo_toml = sh.read_file("Cargo.toml")?.parse::()?; - patch(&mut cargo_toml, "dependencies", &args); - patch(&mut cargo_toml, "dev-dependencies", &args); + edit_dependencies(&mut cargo_toml, "dependencies", &args); + edit_dependencies(&mut cargo_toml, "dev-dependencies", &args); + edit_patch(&mut cargo_toml, &args); if !args.dry_run { sh.write_file("Cargo.toml", cargo_toml.to_string())?; cmd!(sh, "cargo fetch").run()?; + purge_unused_patches(&mut cargo_toml)?; + sh.write_file("Cargo.toml", cargo_toml.to_string())?; + eprintln!("$ cargo xtask set-version"); set_version::main(Default::default())?; } @@ -60,68 +65,180 @@ pub fn main(args: Args) -> Result<()> { Ok(()) } -fn patch(cargo_toml: &mut DocumentMut, table: &str, args: &Args) { - let deps = &mut cargo_toml[table].as_table_mut().unwrap(); +fn edit_dependencies(cargo_toml: &mut DocumentMut, table: &str, args: &Args) { + let deps = cargo_toml[table].as_table_mut().unwrap(); - for (dep_name, dep) in deps - .iter_mut() - .filter(|(key, _)| key.get().starts_with("cairo-lang-") && key != "cairo-lang-macro") - { - let dep_name = dep_name.get(); + for (_, dep) in deps.iter_mut().filter(|(key, _)| is_cairo_crate(key)) { let dep = dep.as_value_mut().unwrap(); - // Start with expanded form: { version = "X" } - let mut new_dep = InlineTable::new(); + // In `[dependencies]` and `[dev-dependencies]` tables, always use crates.io requirements + // so that downstream (Scarb) can reliably patch them with the `[patch.crates-io]` table. + let mut new_dep = InlineTable::from_iter([("version", match &args.spec.version { + Some(version) => Value::from(version.to_string()), + None => Value::from("*"), + })]); - if let Some(version) = &args.spec.version { - new_dep.insert("version", version.to_string().into()); - } + copy_dependency_features(&mut new_dep, dep); - // Add a Git branch or revision reference if requested. - if args.spec.rev.is_some() || args.spec.branch.is_some() { - new_dep.insert("git", "https://github.com/starkware-libs/cairo".into()); - } + *dep = new_dep.into(); + simplify_dependency_table(dep) + } - if let Some(branch) = &args.spec.branch { - new_dep.insert("branch", branch.as_str().into()); - } + deps.fmt(); + deps.sort_values(); + + eprintln!("[{table}]"); + for (key, dep) in deps.iter().filter(|(key, _)| is_cairo_crate(key)) { + eprintln!("{key} = {dep}"); + } +} + +fn edit_patch(cargo_toml: &mut DocumentMut, args: &Args) { + let patch = cargo_toml["patch"].as_table_mut().unwrap()["crates-io"].as_table_mut().unwrap(); + + // Clear any existing entries for Cairo crates. + patch.retain(|key, _| !is_cairo_crate(key)); + + // Leave this section empty if we are requested to just use a specific version. + if args.spec.rev.is_some() || args.spec.branch.is_some() || args.spec.path.is_some() { + // Patch all Cairo crates that exist, even if this project does not directly depend on them, + // to avoid any duplicates in transient dependencies. + for &dep_name in CAIRO_CRATES { + let mut dep = InlineTable::new(); + + // Add a Git branch or revision reference if requested. + if args.spec.rev.is_some() || args.spec.branch.is_some() { + dep.insert("git", "https://github.com/starkware-libs/cairo".into()); + } + + if let Some(branch) = &args.spec.branch { + dep.insert("branch", branch.as_str().into()); + } + + if let Some(rev) = &args.spec.rev { + dep.insert("rev", rev.as_str().into()); + } + + // Add local path reference if requested. + // For local path sources, Cargo is not looking for crates recursively therefore, we + // need to manually provide full paths to Cairo workspace member crates. + if let Some(path) = &args.spec.path { + dep.insert( + "path", + path.join("crates").join(dep_name).to_string_lossy().into_owned().into(), + ); + } - if let Some(rev) = &args.spec.rev { - new_dep.insert("rev", rev.as_str().into()); + patch.insert(dep_name, dep.into()); } + } + + patch.fmt(); + patch.sort_values(); + + eprintln!("[patch.crates-io]"); + for (key, dep) in patch.iter() { + eprintln!("{key} = {dep}"); + } +} + +/// List of library crates published from the starkware-libs/cairo repository. +/// +/// One can get this list from the `scripts/release_crates.sh` script in that repo. +/// Keep this list sorted for better commit diffs. +const CAIRO_CRATES: &[&str] = &[ + "cairo-lang-casm", + "cairo-lang-compiler", + "cairo-lang-debug", + "cairo-lang-defs", + "cairo-lang-diagnostics", + "cairo-lang-doc", + "cairo-lang-eq-solver", + "cairo-lang-executable", + "cairo-lang-filesystem", + "cairo-lang-formatter", + "cairo-lang-lowering", + "cairo-lang-parser", + "cairo-lang-plugins", + "cairo-lang-proc-macros", + "cairo-lang-project", + "cairo-lang-runnable-utils", + "cairo-lang-runner", + "cairo-lang-semantic", + "cairo-lang-sierra", + "cairo-lang-sierra-ap-change", + "cairo-lang-sierra-gas", + "cairo-lang-sierra-generator", + "cairo-lang-sierra-to-casm", + "cairo-lang-sierra-type-size", + "cairo-lang-starknet", + "cairo-lang-starknet-classes", + "cairo-lang-syntax", + "cairo-lang-syntax-codegen", + "cairo-lang-test-plugin", + "cairo-lang-test-runner", + "cairo-lang-test-utils", + "cairo-lang-utils", +]; + +/// Checks if the given crate is released from the starkware-libs/cairo repository. +fn is_cairo_crate(name: &str) -> bool { + CAIRO_CRATES.contains(&name) +} - // Add local path reference if requested. - // For local path sources, Cargo is not looking for crates recursively therefore, - // we need to manually provide full paths to Cairo workspace member crates. - if let Some(path) = &args.spec.path { - new_dep.insert( - "path", - path.join("crates").join(dep_name).to_string_lossy().into_owned().into(), - ); +/// Copies features from source dependency spec to new dependency table, if exists. +fn copy_dependency_features(dest: &mut InlineTable, src: &Value) { + if let Some(dep) = src.as_inline_table() { + if let Some(features) = dep.get("features") { + dest.insert("features", features.clone()); } + } +} - // Sometimes we might specify extra features. Let's preserve these. - if let Some(dep) = dep.as_inline_table() { - if let Some(features) = dep.get("features") { - new_dep.insert("features", features.clone()); +/// Simplifies a `{ version = "V" }` dependency spec to shorthand `"V"` if possible. +fn simplify_dependency_table(dep: &mut Value) { + *dep = match mem::replace(dep, false.into()) { + Value::InlineTable(mut table) => { + if table.len() == 1 { + table.remove("version").unwrap_or_else(|| table.into()) + } else { + table.into() } } - // Simplify { version = "X" } to "X" if possible. - let new_dep: Value = if new_dep.len() == 1 { - new_dep.remove("version").unwrap_or_else(|| new_dep.into()) - } else { - new_dep.into() - }; - - *dep = new_dep; + dep => dep, } +} - deps.fmt(); - deps.sort_values(); +/// Remove any unused patches from the `[patch.crates-io]` table. +/// +/// We are adding patch entries for **all** Cairo crates existing, and some may end up being unused. +/// Cargo is emitting warnings about unused patches and keeps a record of them in the `Cargo.lock`. +/// The goal of this function is to resolve these warnings. +fn purge_unused_patches(cargo_toml: &mut DocumentMut) -> Result<()> { + let sh = Shell::new()?; + let cargo_lock = sh.read_file("Cargo.lock")?.parse::()?; - eprintln!("[{table}]"); - for (key, dep) in deps.iter().filter(|(key, _)| key.starts_with("cairo-lang-")) { - eprintln!("{key} = {dep}"); + if let Some(unused_patches) = find_unused_patches(&cargo_lock) { + let patch = + cargo_toml["patch"].as_table_mut().unwrap()["crates-io"].as_table_mut().unwrap(); + + // Remove any patches that are not for Cairo crates. + patch.retain(|key, _| !unused_patches.contains(&key.to_owned())); } + + Ok(()) +} + +/// Extracts names of unused patches from the `[[patch.unused]]` array from the `Cargo.lock` file. +fn find_unused_patches(cargo_lock: &DocumentMut) -> Option> { + Some( + cargo_lock + .get("patch")? + .get("unused")? + .as_array_of_tables()? + .iter() + .flat_map(|table| Some(table.get("name")?.as_str()?.to_owned())) + .collect(), + ) }