From eb3b386ee4a4cc0bdbb3c3358e036e0e1f9c0b06 Mon Sep 17 00:00:00 2001 From: konsti Date: Thu, 7 Nov 2024 23:38:56 +0100 Subject: [PATCH] Add codespeed for continuous profiling (and sudoku) (#271) * Add codespeed for continuous (and sudoku) While pubgrub's performance is critical for both uv and cargo, there currently no benchmarking happening in the repository. [Codspeed](https://codspeed.io) runs our benchmarks with instruction counting, reporting more numbers with less variance than wall time. We get feedback on every PR, can see trends over time and there are flamegraphs and flamegraph diffs in the web view. We've made good experiences with it in both ruff and uv. With codspeed installed, we can start adding real-world benchmarks for uv and cargo to the pubgrub repo, and then optimize those. * serde --- .github/workflows/benchmarks.yml | 29 +++++ Cargo.lock | 133 ++++++++++++++++++++--- Cargo.toml | 6 +- benches/sudoku.rs | 177 +++++++++++++++++++++++++++++++ 4 files changed, 327 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benches/sudoku.rs diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..1b060ccc --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,29 @@ +name: Benchmarks (CodSpeed) + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + + - name: Build the benchmark target(s) + run: cargo codspeed build --features serde + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v3 + with: + run: cargo codspeed run diff --git a/Cargo.lock b/Cargo.lock index 5155c266..4c49b04f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,7 +52,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -62,7 +62,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -177,12 +177,44 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "codspeed" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "450a0e9df9df1c154156f4344f99d8f6f6e69d0fc4de96ef6e2e68b2ec3bce97" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb1a6cb9c20e177fde58cdef97c1c7c9264eb1424fe45c4fccedc2fb078a569" +dependencies = [ + "codspeed", + "colored", + "criterion", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "criterion" version = "0.5.1" @@ -292,7 +324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -364,7 +396,7 @@ checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -568,7 +600,7 @@ dependencies = [ name = "pubgrub" version = "0.2.1" dependencies = [ - "criterion", + "codspeed-criterion-compat", "env_logger", "indexmap", "log", @@ -720,7 +752,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -772,11 +804,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa 1.0.10", + "memchr", "ryu", "serde", ] @@ -833,7 +866,7 @@ dependencies = [ "cfg-if", "fastrand", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1107,13 +1140,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -1122,51 +1179,93 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.4" diff --git a/Cargo.toml b/Cargo.toml index caef5328..1b33cc77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0" version-ranges = { version = "0.1.0", path = "version-ranges" } [dev-dependencies] -criterion = "0.5" +criterion = { version = "2.7.2", package = "codspeed-criterion-compat" } env_logger = "0.11.5" proptest = "1.5.0" ron = "=0.9.0-alpha.0" @@ -46,3 +46,7 @@ serde = ["dep:serde", "version-ranges/serde"] name = "large_case" harness = false required-features = ["serde"] + +[[bench]] +name = "sudoku" +harness = false diff --git a/benches/sudoku.rs b/benches/sudoku.rs new file mode 100644 index 00000000..47cd2e2f --- /dev/null +++ b/benches/sudoku.rs @@ -0,0 +1,177 @@ +//! A sudoku solver. +// SPDX-License-Identifier: MPL-2.0 + +use std::fmt; + +use pubgrub::{ + resolve, DefaultStringReporter, OfflineDependencyProvider, PubGrubError, Range, Reporter, + SelectedDependencies, +}; +use version_ranges::Ranges; + +use criterion::*; + +/// The size of a box in the board. +const BOARD_BASE: u8 = 3; +/// The size of the board. +const BOARD_SIZE: u8 = BOARD_BASE * BOARD_BASE; + +type DP = OfflineDependencyProvider>; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum SudokuPackage { + /// Add all known fields. + Root, + /// Version is the value of the cell. + Cell { row: u8, col: u8 }, +} + +impl fmt::Display for SudokuPackage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SudokuPackage::Root => f.write_str("root"), + SudokuPackage::Cell { row, col } => { + write!(f, "({col}, {row})") + } + } + } +} + +fn from_board(b: &str) -> Vec<(SudokuPackage, Range)> { + let mut out = vec![]; + for (row, line) in b + .trim() + .lines() + .map(str::trim) + .filter(|l| !l.starts_with('-')) + .enumerate() + { + for (col, val) in line + .split_ascii_whitespace() + .filter(|c| !c.starts_with('|')) + .enumerate() + { + if let Some(val) = val.chars().next().unwrap().to_digit(10) { + out.push(( + SudokuPackage::Cell { + row: (row + 1).try_into().unwrap(), + col: (col + 1).try_into().unwrap(), + }, + Range::singleton(val as u8), + )); + } + } + } + out +} + +/// Encode all the exclusions from assigning a cell to a value +fn encode_constraints( + dependency_provider: &mut OfflineDependencyProvider>, +) { + for row in 1..=BOARD_SIZE { + for col in 1..=BOARD_SIZE { + for val in 1..=BOARD_SIZE { + let mut deps = vec![]; + // A number may only occur once in a row + for row_ in 1..=BOARD_SIZE { + if row_ == row { + continue; + } + deps.push(( + SudokuPackage::Cell { row: row_, col }, + Range::singleton(val).complement(), + )) + } + // A number may only occur once in a col + for col_ in 1..=BOARD_SIZE { + if col_ == col { + continue; + } + deps.push(( + SudokuPackage::Cell { row, col: col_ }, + Range::singleton(val).complement(), + )) + } + // A number may only occur once in a box + let box_base_row = row - ((row - 1) % BOARD_BASE); + let box_base_col = col - ((col - 1) % BOARD_BASE); + for row_ in box_base_row..box_base_row + BOARD_BASE { + for col_ in box_base_col..box_base_col + BOARD_BASE { + if col_ == col && row_ == row { + continue; + } + deps.push(( + SudokuPackage::Cell { + row: row_, + col: col_, + }, + Range::singleton(val).complement(), + )) + } + } + let name = SudokuPackage::Cell { row, col }; + dependency_provider.add_dependencies(name, val, deps) + } + } + } +} + +fn solve(board: Vec<(SudokuPackage, Ranges)>) -> SelectedDependencies { + let mut dependency_provider = DP::new(); + encode_constraints(&mut dependency_provider); + dependency_provider.add_dependencies(SudokuPackage::Root, 1, board); + match resolve(&dependency_provider, SudokuPackage::Root, 1) { + Ok(sol) => sol, + Err(PubGrubError::NoSolution(mut derivation_tree)) => { + derivation_tree.collapse_no_versions(); + eprintln!("{}", DefaultStringReporter::report(&derivation_tree)); + std::process::exit(1); + } + Err(err) => panic!("{:?}", err), + } +} + +fn bench_solve(c: &mut Criterion) { + let easy = from_board( + r#" + 5 3 _ | _ 7 _ | _ _ _ + 6 _ _ | 1 9 5 | _ _ _ + _ 9 8 | _ _ _ | _ 6 _ + -------+-------+------- + 8 5 9 | _ 6 1 | 4 2 3 + 4 2 6 | 8 5 3 | 7 9 1 + 7 1 3 | 9 2 4 | 8 5 6 + -------+-------+------- + _ 6 _ | _ _ _ | 2 8 _ + _ _ _ | 4 1 9 | _ _ 5 + _ _ _ | _ 8 6 | 1 7 9"#, + ); + c.bench_function("sudoku-easy", |b| { + b.iter(|| { + solve(black_box(easy.clone())); + }) + }); + let hard = from_board( + r#" + 5 3 _ | _ 7 _ | _ _ _ + 6 _ _ | 1 9 5 | _ _ _ + _ 9 8 | _ _ _ | _ 6 _ + -------+-------+------- + 8 _ _ | _ 6 _ | _ _ 3 + 4 _ _ | 8 _ 3 | _ _ 1 + 7 _ _ | _ 2 _ | _ _ 6 + -------+-------+------- + _ 6 _ | _ _ _ | 2 8 _ + _ _ _ | 4 1 9 | _ _ 5 + _ _ _ | _ 8 _ | _ 7 9"#, + ); + c.bench_function("sudoku-hard", |b| { + b.iter(|| { + solve(black_box(hard.clone())); + }) + }); +} + +criterion_group!(benches, bench_solve); +criterion_main!(benches);