Skip to content

Commit

Permalink
Use patch-based Cairo dependencies specification (#33)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mkaput
mkaput authored Dec 11, 2024
1 parent c73f7d5 commit 3d6f933
Show file tree
Hide file tree
Showing 3 changed files with 246 additions and 64 deletions.
16 changes: 16 additions & 0 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,22 @@ to push further changes to your commits.
Our policy is to not accept PRs that only fix typos in the documentation and code.
We appreciate your effort, but we encourage you to focus on bugs and features instead.

## Tips

### Testing custom Cairo compiler changes

Sometimes you may happen to work on a feature to the Cairo compiler,
and you would like to test how it works in CairoLS.

We have a script that edits the `Cargo.toml` file to use a local checkout of the Cairo compiler.
To use this tool, run:

```shell
cargo xtask set-cairo-version --path ../path/to/cairo
```

And then you can `cargo build` CairoLS with your custom Cairo compiler changes.

---

Thanks! ❤️ ❤️ ❤️
Expand Down
79 changes: 64 additions & 15 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,38 @@ members = ["xtask"]
[features]
testing = []

# Managing dependencies on crates from starkware-libs/cairo repository:
#
# The Cairo compiler is made of a bunch of crates that inter-depend on each other and have
# synchronised versioning.
# It is very important to use a single revision of these crates in the entire Cairo toolchain,
# which consists of Cairo compiler, Scarb, CairoLS and other tools.
# The toolchain is eventually built by Scarb, which depends on everything other as regular crates.
# To ensure that all crates in the toolchain use the same revision of Cairo crates, we use a patch
# mechanism that Cargo provides.
# Because Cargo requires patches to change the crate source, we have an unspoken contract that
# all tools *always* depend on some crates.io versions of Cairo crates and Scarb uses
# [patch.crates.io] table to set final git revision for everything.
#
# To keep our Cargo.toml following this contract, always use `cargo xtask set-cairo-version`
# for manipulating these dependencies.
[dependencies]
anyhow = "1"
cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-compiler = "*"
cairo-lang-defs = "*"
cairo-lang-diagnostics = "*"
cairo-lang-doc = "*"
cairo-lang-filesystem = "*"
cairo-lang-formatter = "*"
cairo-lang-lowering = "*"
cairo-lang-macro = "0.1.1"
cairo-lang-parser = "*"
cairo-lang-project = "*"
cairo-lang-semantic = "*"
cairo-lang-starknet = "*"
cairo-lang-syntax = "*"
cairo-lang-test-plugin = "*"
cairo-lang-utils = "*"
convert_case = "0.6.0"
crossbeam = "0.8.4"
governor = { version = "0.7.0", default-features = false, features = ["std", "quanta"] }
Expand Down Expand Up @@ -59,12 +74,46 @@ libc = "0.2.167"

[dev-dependencies]
assert_fs = "1.1"
cairo-lang-test-utils = { version = "*", features = ["testing"] }
cairo-language-server = { path = ".", features = ["testing"] }
cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174", features = ["testing"] }
pathdiff = "0.2"
pretty_assertions = "1.4.0"
test-log = "0.2.16"

# Here we specify real dependency specifications for Cairo crates *if* currently we want to use
# a particular unreleased commit (which is frequent mid-development).
# We list all Cairo crates that go into CairoLS's compilation unit even if LS itself does not depend
# on some of them directly.
# This ensures no duplicate instances of Cairo crates are pulled in by mistake.
[patch.crates-io]
cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-defs = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-diagnostics = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-doc = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-eq-solver = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-plugins = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-proc-macros = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra-ap-change = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra-gas = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra-generator = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra-to-casm = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-sierra-type-size = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-starknet = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-starknet-classes = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-syntax = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-syntax-codegen = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-test-plugin = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-test-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }
cairo-lang-utils = { git = "https://github.com/starkware-libs/cairo", rev = "0b86ece404b0922b76caca5d07a94ed41407f174" }

# The profile used for CI in pull requests.
# External dependencies are built with optimisation enabled,
Expand Down
215 changes: 166 additions & 49 deletions xtask/src/set_cairo_version.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::mem;
use std::path::PathBuf;

use anyhow::Result;
Expand All @@ -8,7 +9,7 @@ use xshell::{Shell, cmd};

use crate::set_version;

/// Update `cairo-lang-*` crates in a proper way.
/// Update `cairo-lang-*` crates properly.
#[derive(Parser)]
pub struct Args {
#[command(flatten)]
Expand Down Expand Up @@ -45,83 +46,199 @@ pub fn main(args: Args) -> Result<()> {

let mut cargo_toml = sh.read_file("Cargo.toml")?.parse::<DocumentMut>()?;

patch(&mut cargo_toml, "dependencies", &args);
patch(&mut cargo_toml, "dev-dependencies", &args);
edit_dependencies(&mut cargo_toml, "dependencies", &args);
edit_dependencies(&mut cargo_toml, "dev-dependencies", &args);
edit_patch(&mut cargo_toml, &args);

if !args.dry_run {
sh.write_file("Cargo.toml", cargo_toml.to_string())?;

cmd!(sh, "cargo fetch").run()?;

purge_unused_patches(&mut cargo_toml)?;
sh.write_file("Cargo.toml", cargo_toml.to_string())?;

eprintln!("$ cargo xtask set-version");
set_version::main(Default::default())?;
}

Ok(())
}

fn patch(cargo_toml: &mut DocumentMut, table: &str, args: &Args) {
let deps = &mut cargo_toml[table].as_table_mut().unwrap();
fn edit_dependencies(cargo_toml: &mut DocumentMut, table: &str, args: &Args) {
let deps = cargo_toml[table].as_table_mut().unwrap();

for (dep_name, dep) in deps
.iter_mut()
.filter(|(key, _)| key.get().starts_with("cairo-lang-") && key != "cairo-lang-macro")
{
let dep_name = dep_name.get();
for (_, dep) in deps.iter_mut().filter(|(key, _)| is_cairo_crate(key)) {
let dep = dep.as_value_mut().unwrap();

// Start with expanded form: { version = "X" }
let mut new_dep = InlineTable::new();
// In `[dependencies]` and `[dev-dependencies]` tables, always use crates.io requirements
// so that downstream (Scarb) can reliably patch them with the `[patch.crates-io]` table.
let mut new_dep = InlineTable::from_iter([("version", match &args.spec.version {
Some(version) => Value::from(version.to_string()),
None => Value::from("*"),
})]);

if let Some(version) = &args.spec.version {
new_dep.insert("version", version.to_string().into());
}
copy_dependency_features(&mut new_dep, dep);

// Add a Git branch or revision reference if requested.
if args.spec.rev.is_some() || args.spec.branch.is_some() {
new_dep.insert("git", "https://github.com/starkware-libs/cairo".into());
}
*dep = new_dep.into();
simplify_dependency_table(dep)
}

if let Some(branch) = &args.spec.branch {
new_dep.insert("branch", branch.as_str().into());
}
deps.fmt();
deps.sort_values();

eprintln!("[{table}]");
for (key, dep) in deps.iter().filter(|(key, _)| is_cairo_crate(key)) {
eprintln!("{key} = {dep}");
}
}

fn edit_patch(cargo_toml: &mut DocumentMut, args: &Args) {
let patch = cargo_toml["patch"].as_table_mut().unwrap()["crates-io"].as_table_mut().unwrap();

// Clear any existing entries for Cairo crates.
patch.retain(|key, _| !is_cairo_crate(key));

// Leave this section empty if we are requested to just use a specific version.
if args.spec.rev.is_some() || args.spec.branch.is_some() || args.spec.path.is_some() {
// Patch all Cairo crates that exist, even if this project does not directly depend on them,
// to avoid any duplicates in transient dependencies.
for &dep_name in CAIRO_CRATES {
let mut dep = InlineTable::new();

// Add a Git branch or revision reference if requested.
if args.spec.rev.is_some() || args.spec.branch.is_some() {
dep.insert("git", "https://github.com/starkware-libs/cairo".into());
}

if let Some(branch) = &args.spec.branch {
dep.insert("branch", branch.as_str().into());
}

if let Some(rev) = &args.spec.rev {
dep.insert("rev", rev.as_str().into());
}

// Add local path reference if requested.
// For local path sources, Cargo is not looking for crates recursively therefore, we
// need to manually provide full paths to Cairo workspace member crates.
if let Some(path) = &args.spec.path {
dep.insert(
"path",
path.join("crates").join(dep_name).to_string_lossy().into_owned().into(),
);
}

if let Some(rev) = &args.spec.rev {
new_dep.insert("rev", rev.as_str().into());
patch.insert(dep_name, dep.into());
}
}

patch.fmt();
patch.sort_values();

eprintln!("[patch.crates-io]");
for (key, dep) in patch.iter() {
eprintln!("{key} = {dep}");
}
}

/// List of library crates published from the starkware-libs/cairo repository.
///
/// One can get this list from the `scripts/release_crates.sh` script in that repo.
/// Keep this list sorted for better commit diffs.
const CAIRO_CRATES: &[&str] = &[
"cairo-lang-casm",
"cairo-lang-compiler",
"cairo-lang-debug",
"cairo-lang-defs",
"cairo-lang-diagnostics",
"cairo-lang-doc",
"cairo-lang-eq-solver",
"cairo-lang-executable",
"cairo-lang-filesystem",
"cairo-lang-formatter",
"cairo-lang-lowering",
"cairo-lang-parser",
"cairo-lang-plugins",
"cairo-lang-proc-macros",
"cairo-lang-project",
"cairo-lang-runnable-utils",
"cairo-lang-runner",
"cairo-lang-semantic",
"cairo-lang-sierra",
"cairo-lang-sierra-ap-change",
"cairo-lang-sierra-gas",
"cairo-lang-sierra-generator",
"cairo-lang-sierra-to-casm",
"cairo-lang-sierra-type-size",
"cairo-lang-starknet",
"cairo-lang-starknet-classes",
"cairo-lang-syntax",
"cairo-lang-syntax-codegen",
"cairo-lang-test-plugin",
"cairo-lang-test-runner",
"cairo-lang-test-utils",
"cairo-lang-utils",
];

/// Checks if the given crate is released from the starkware-libs/cairo repository.
fn is_cairo_crate(name: &str) -> bool {
CAIRO_CRATES.contains(&name)
}

// Add local path reference if requested.
// For local path sources, Cargo is not looking for crates recursively therefore,
// we need to manually provide full paths to Cairo workspace member crates.
if let Some(path) = &args.spec.path {
new_dep.insert(
"path",
path.join("crates").join(dep_name).to_string_lossy().into_owned().into(),
);
/// Copies features from source dependency spec to new dependency table, if exists.
fn copy_dependency_features(dest: &mut InlineTable, src: &Value) {
if let Some(dep) = src.as_inline_table() {
if let Some(features) = dep.get("features") {
dest.insert("features", features.clone());
}
}
}

// Sometimes we might specify extra features. Let's preserve these.
if let Some(dep) = dep.as_inline_table() {
if let Some(features) = dep.get("features") {
new_dep.insert("features", features.clone());
/// Simplifies a `{ version = "V" }` dependency spec to shorthand `"V"` if possible.
fn simplify_dependency_table(dep: &mut Value) {
*dep = match mem::replace(dep, false.into()) {
Value::InlineTable(mut table) => {
if table.len() == 1 {
table.remove("version").unwrap_or_else(|| table.into())
} else {
table.into()
}
}

// Simplify { version = "X" } to "X" if possible.
let new_dep: Value = if new_dep.len() == 1 {
new_dep.remove("version").unwrap_or_else(|| new_dep.into())
} else {
new_dep.into()
};

*dep = new_dep;
dep => dep,
}
}

deps.fmt();
deps.sort_values();
/// Remove any unused patches from the `[patch.crates-io]` table.
///
/// We are adding patch entries for **all** Cairo crates existing, and some may end up being unused.
/// Cargo is emitting warnings about unused patches and keeps a record of them in the `Cargo.lock`.
/// The goal of this function is to resolve these warnings.
fn purge_unused_patches(cargo_toml: &mut DocumentMut) -> Result<()> {
let sh = Shell::new()?;
let cargo_lock = sh.read_file("Cargo.lock")?.parse::<DocumentMut>()?;

eprintln!("[{table}]");
for (key, dep) in deps.iter().filter(|(key, _)| key.starts_with("cairo-lang-")) {
eprintln!("{key} = {dep}");
if let Some(unused_patches) = find_unused_patches(&cargo_lock) {
let patch =
cargo_toml["patch"].as_table_mut().unwrap()["crates-io"].as_table_mut().unwrap();

// Remove any patches that are not for Cairo crates.
patch.retain(|key, _| !unused_patches.contains(&key.to_owned()));
}

Ok(())
}

/// Extracts names of unused patches from the `[[patch.unused]]` array from the `Cargo.lock` file.
fn find_unused_patches(cargo_lock: &DocumentMut) -> Option<Vec<String>> {
Some(
cargo_lock
.get("patch")?
.get("unused")?
.as_array_of_tables()?
.iter()
.flat_map(|table| Some(table.get("name")?.as_str()?.to_owned()))
.collect(),
)
}

0 comments on commit 3d6f933

Please sign in to comment.