hyperfine.yml

name: Hyperfine Benchmark

on:
  pull_request:
    branches: [ '**' ]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:
  CARGO_TERM_COLOR: always

jobs:
  build-programs:
    strategy:
      matrix:
        branch: [ base, head ]
    name: Build Cairo programs for ${{ matrix.branch }}
    runs-on: ubuntu-22.04
    outputs:
      benchmark-hashes-base: ${{ steps.export-hashes.outputs.benchmark-hashes-base }}
      benchmark-hashes-head: ${{ steps.export-hashes.outputs.benchmark-hashes-head }}
    steps:
      - name: Checkout base commit
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request[matrix.branch].sha }}

      - name: Fetch from cache
        uses: actions/cache@v3
        id: cache
        with:
          path: ${{ matrix.branch }}_programs/*.json
          key: benchmarks-${{ matrix.branch }}-${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}
          restore-keys: benchmarks-${{ matrix.branch }}-

      - name: Install Python
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'

      - name: Install Cairo compiler
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: pip install -r requirements.txt

      - name: Build programs
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          make -j cairo_bench_programs
          mkdir -p ${{ matrix.branch }}_programs
          cp cairo_programs/benchmarks/*.json ${{ matrix.branch }}_programs

      - name: Export benchmark hashes
        id: export-hashes
        run:
          echo "benchmark-hashes-${{ matrix.branch }}=${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}" >> "$GITHUB_OUTPUT"


  build-binaries:
    strategy:
      matrix:
        branch: [ base, head ]
    name: Build cairo-vm-cli for ${{ matrix.branch }}
    runs-on: ubuntu-22.04
    steps:
      - name: Populate cache
        uses: actions/cache@v3
        id: cache 
        with:
          path: bin/cairo-vm-cli-${{ matrix.branch }}
          key: binary-${{ github.event.pull_request[matrix.branch].sha }}

      - name: Install Rust
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: dtolnay/rust-toolchain@1.80.0

      - name: Checkout
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request[matrix.branch].sha }}

      - name: Fetch Rust cache
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: Swatinem/rust-cache@v2

      - name: Build binary
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          cargo b --release -p cairo-vm-cli
          mkdir bin
          cp target/release/cairo-vm-cli bin/cairo-vm-cli-${{ matrix.branch }}


  run-hyperfine:
    strategy:
      matrix:
        program_state: [ modified, unmodified ]
    name: Run benchmarks for ${{ matrix.program_state }} programs
    needs: [ build-programs, build-binaries ]
    runs-on: ubuntu-22.04
    steps:
    - name: Install Hyperfine
      uses: taiki-e/install-action@v2
      with:
        tool: hyperfine@1.16

    - name: Fetch base binary
      uses: actions/cache/restore@v3
      with:
        path: bin/cairo-vm-cli-base
        key: binary-${{ github.event.pull_request.base.sha }}

    - name: Fetch HEAD binary
      uses: actions/cache/restore@v3
      with:
        path: bin/cairo-vm-cli-head
        key: binary-${{ github.event.pull_request.head.sha }}

    - name: Fetch base programs
      uses: actions/cache/restore@v3
      with:
        path: base_programs/*.json
        key: benchmarks-base-${{ needs.build-programs.outputs.benchmark-hashes-base }}

    - name: Fetch head programs
      uses: actions/cache/restore@v3
      with:
        path: head_programs/*.json
        key: benchmarks-head-${{ needs.build-programs.outputs.benchmark-hashes-head }}

    - name: Benchmark ${{ matrix.program_state }} programs
      id: run-benchmarks
      run: |
        sudo swapoff -a
        mkdir target_programs
        if [ 'modified' = ${{ matrix.program_state }} ]; then
          BINS=head
          for f in head_programs/*.json; do
            # Only run new or modified benchmarks
            if ! cmp -s ${f/head/base} $f; then
              cp $f target_programs/
            fi
          done
        else
          BINS="base,head"
          for f in base_programs/*.json; do
            # Only run unmodified benchmarks
            if cmp -s ${f/base/head} $f; then
              cp $f target_programs/
            fi
          done
        fi
        find target_programs -name '*.json' -exec basename -s .json '{}' '+' | \
        sort | xargs -I '{program}' \
        hyperfine -N -r 10 --export-markdown "target_programs/{program}.md" \
        -L bin "$BINS" -n "{bin} {program}" \
        -s "cat ./bin/cairo-vm-cli-{bin} target_programs/{program}.json" \
        "./bin/cairo-vm-cli-{bin} --proof_mode --layout starknet_with_keccak \
         --memory_file /dev/null --trace_file /dev/null target_programs/{program}.json"
        echo "benchmark_count=$(ls target_programs/*.md | wc -l)" >> $GITHUB_OUTPUT

    - name: Print tables
      if: steps.run-benchmarks.outputs.benchmark_count != 0
      run: |
        {
          echo "Benchmark Results for ${{ matrix.program_state }} programs :rocket:"
          for f in target_programs/*.md; do
            echo
            cat $f
          done
        } | tee -a comment_body.md

    - name: Find comment
      if: ${{ steps.run-benchmarks.outputs.benchmark_count != 0 }}
      uses: peter-evans/find-comment@v2
      id: fc
      with:
        issue-number: ${{ github.event.pull_request.number }}
        comment-author: 'github-actions[bot]'
        body-includes: Benchmark Results for ${{ matrix.program_state }} programs

    - name: Create comment
      if: steps.fc.outputs.comment-id == '' && steps.run-benchmarks.outputs.benchmark_count != 0
      uses: peter-evans/create-or-update-comment@v3
      with:
        issue-number: ${{ github.event.pull_request.number }}
        body-path: comment_body.md

    - name: Update comment
      if: steps.fc.outputs.comment-id != '' && steps.run-benchmarks.outputs.benchmark_count != 0
      uses: peter-evans/create-or-update-comment@v3
      with:
        comment-id: ${{ steps.fc.outputs.comment-id }}
        body-path: comment_body.md
        edit-mode: replace