edr-benchmark.yml

name: EDR Benchmark

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - "**"
  workflow_dispatch:

defaults:
  run:
    working-directory: crates/tools/js/benchmark

concurrency:
  group: ${{github.workflow}}-${{github.ref}}
  # Don't cancel in progress jobs in main
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  js-benchmark:
    name: Run JS scenario runner benchmark
    environment: github-action-benchmark
    runs-on: self-hosted
    # Only run for trusted collaborators since third-parties could run malicious code on the self-hosted benchmark runner.
    if: github.ref == 'refs/heads/main' || github.repository == github.event.pull_request.head.repo.full_name
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-node
      - uses: ./.github/actions/setup-rust

      - name: Install package
        run: pnpm install --frozen-lockfile --prefer-offline

      - name: Run benchmark tests
        run: pnpm test

      - name: Run benchmark
        run: pnpm run -s benchmark

      - name: Validate regressions
        run: pnpm run -s verify

      - name: Generate report for github-action-benchmark
        run: pnpm run -s report | tee report.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: customSmallerIsBetter
          output-file-path: crates/tools/js/benchmark/report.json
          gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
          gh-pages-branch: main
          benchmark-data-dir-path: bench
          github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
          # Only save the data for main branch pushes. For PRs we only compare
          auto-push: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' }}
          alert-threshold: "110%"
          # Only fail on pull requests, don't break CI in main
          fail-on-alert: ${{ github.event_name == 'pull_request' }}
          # Enable Job Summary for PRs
          summary-always: true
          max-items-in-chart: 1000