From 3d666a1af3e6b9ae8811685ad674b0a444ca0286 Mon Sep 17 00:00:00 2001
From: Shahar Papini <43779613+spapinistarkware@users.noreply.github.com>
Date: Sun, 17 Mar 2024 11:23:31 +0200
Subject: [PATCH] Benchmarks in CI (#485)

---
 .github/runners/Dockerfile              |  3 ++-
 .github/workflows/benchmarks-pages.yaml | 35 +++++++++++++++++++++++++
 .github/workflows/ci.yaml               | 31 +++++++++++++++++++---
 Cargo.toml                              |  3 +++
 benches/bit_rev.rs                      |  8 +++---
 benches/fft.rs                          |  4 +--
 benches/field.rs                        | 19 +++++++++-----
 scripts/bench.sh                        |  2 +-
 8 files changed, 86 insertions(+), 19 deletions(-)
 create mode 100644 .github/workflows/benchmarks-pages.yaml

diff --git a/.github/runners/Dockerfile b/.github/runners/Dockerfile
index 0ad10fe00..00a68489e 100644
--- a/.github/runners/Dockerfile
+++ b/.github/runners/Dockerfile
@@ -4,7 +4,8 @@ USER root
 RUN apt update && \
     apt install -y \
     build-essential \
-    curl
+    curl \
+    git
 
 USER runner
 CMD /bin/bash
diff --git a/.github/workflows/benchmarks-pages.yaml b/.github/workflows/benchmarks-pages.yaml
new file mode 100644
index 000000000..3672b6c2c
--- /dev/null
+++ b/.github/workflows/benchmarks-pages.yaml
@@ -0,0 +1,35 @@
+name: 
+
+on:
+  push:
+    branches:
+      - dev
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  run-avx-bench:
+    runs-on: avx
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2024-01-04
+      - name: Run benchmark
+        run: ./scripts/bench.sh -- --output-format bencher | tee output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'cargo'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a9c629ece..ee8a05b5c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -3,10 +3,7 @@ name: CI
 on:
   push:
     branches:
-      - main
-      - main-v[0-9].**
-    tags:
-      - v[0-9].**
+      - dev
 
   pull_request:
     types:
@@ -51,6 +48,32 @@ jobs:
           toolchain: nightly-2024-01-04
       - uses: Swatinem/rust-cache@v2
       - run: ./scripts/test_avx.sh
+  
+  run-avx-bench:
+    runs-on: avx
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2024-01-04
+      - name: Run benchmark
+        run: ./scripts/bench.sh -- --output-format bencher | tee output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'cargo'
+          output-file-path: output.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: true
+          summary-always: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          alert-comment-cc-users: '@spapinistarkware'
 
   run-tests:
     runs-on: ubuntu-latest
diff --git a/Cargo.toml b/Cargo.toml
index 38612a10d..cf763026a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,9 @@ bytemuck = { version = "1.14.3", features = ["derive"] }
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = { version = "0.8.5", features = ["small_rng"] }
 
+[lib]
+bench = false
+
 [lints.rust]
 warnings = "deny"
 future-incompatible = "deny"
diff --git a/benches/bit_rev.rs b/benches/bit_rev.rs
index 1192b3013..4595dcb0f 100644
--- a/benches/bit_rev.rs
+++ b/benches/bit_rev.rs
@@ -6,12 +6,12 @@ use criterion::Criterion;
 pub fn cpu_bit_rev(c: &mut criterion::Criterion) {
     use stwo::core::fields::m31::BaseField;
 
-    const SIZE: usize = 1 << 28;
+    const SIZE: usize = 1 << 24;
     let mut data: Vec<_> = (0..SIZE as u32)
         .map(BaseField::from_u32_unchecked)
         .collect();
 
-    c.bench_function("cpu bit_rev", |b| {
+    c.bench_function("cpu bit_rev 24bit", |b| {
         b.iter(|| {
             stwo::core::utils::bit_reverse(&mut data);
         })
@@ -29,7 +29,7 @@ pub fn avx512_bit_rev(c: &mut criterion::Criterion) {
         return;
     }
 
-    const SIZE: usize = 1 << 28;
+    const SIZE: usize = 1 << 26;
     let data: Vec<_> = (0..SIZE as u32)
         .map(BaseField::from_u32_unchecked)
         .collect();
@@ -39,7 +39,7 @@ pub fn avx512_bit_rev(c: &mut criterion::Criterion) {
         .map(PackedBaseField::from_array)
         .collect();
 
-    c.bench_function("avx bit_rev", |b| {
+    c.bench_function("avx bit_rev 26bit", |b| {
         b.iter(|| {
             bit_reverse_m31(cast_slice_mut(&mut data[..]));
         })
diff --git a/benches/fft.rs b/benches/fft.rs
index fe49dbb4b..bb221c5b2 100644
--- a/benches/fft.rs
+++ b/benches/fft.rs
@@ -13,7 +13,7 @@ pub fn avx512_ifft(c: &mut criterion::Criterion) {
         return;
     }
 
-    const LOG_SIZE: u32 = 28;
+    const LOG_SIZE: u32 = 26;
     let domain = CanonicCoset::new(LOG_SIZE).circle_domain();
     let values = (0..domain.size())
         .map(|i| BaseField::from_u32_unchecked(i as u32))
@@ -23,7 +23,7 @@ pub fn avx512_ifft(c: &mut criterion::Criterion) {
     let mut values = BaseFieldVec::from_iter(values);
     let twiddle_dbls = get_itwiddle_dbls(domain.half_coset);
 
-    c.bench_function("avx ifft", |b| {
+    c.bench_function("avx ifft 26bit", |b| {
         b.iter(|| unsafe {
             ifft::ifft(
                 std::mem::transmute(values.data.as_mut_ptr()),
diff --git a/benches/field.rs b/benches/field.rs
index d3aaacf7f..564ed4cf7 100644
--- a/benches/field.rs
+++ b/benches/field.rs
@@ -1,3 +1,4 @@
+use criterion::Criterion;
 use rand::rngs::ThreadRng;
 use rand::Rng;
 use stwo::core::fields::cm31::CM31;
@@ -187,17 +188,21 @@ pub fn avx512_m31_operations_bench(c: &mut criterion::Criterion) {
 
 #[cfg(target_arch = "x86_64")]
 criterion::criterion_group!(
-    m31_benches,
-    m31_operations_bench,
-    avx512_m31_operations_bench
+    name=m31_benches;
+    config = Criterion::default().sample_size(10);
+    targets=
+        m31_operations_bench,
+        avx512_m31_operations_bench
 );
 #[cfg(not(target_arch = "x86_64"))]
 criterion::criterion_group!(m31_benches, m31_operations_bench);
 
 criterion::criterion_group!(
-    field_comparison,
-    m31_operations_bench,
-    cm31_operations_bench,
-    qm31_operations_bench
+    name=field_comparison;
+    config = Criterion::default().sample_size(10);
+    targets=
+        m31_operations_bench,
+        cm31_operations_bench,
+        qm31_operations_bench
 );
 criterion::criterion_main!(field_comparison, m31_benches);
diff --git a/scripts/bench.sh b/scripts/bench.sh
index 0ad7446a0..669564b42 100755
--- a/scripts/bench.sh
+++ b/scripts/bench.sh
@@ -2,4 +2,4 @@
 # Can be used as a drop in replacement for `cargo bench`.
 # For example, `./scripts/bench.sh` will run all benchmarks.
 # or `./scripts/bench.sh M31` will run only the M31 benchmarks.
-RUSTFLAGS="-Awarnings -C target-cpu=native -C target-feature=+avx512f -C opt-level=2" cargo bench "$@"
+RUSTFLAGS="-Awarnings -C target-cpu=native -C target-feature=+avx512f -C opt-level=2" cargo bench $@