Solidity Tests feature branch CI

.cargo/config.toml

@@ -3,3 +3,4 @@ gen-execution-api       = "run --bin tools -- gen-execution-api"
 scenario                = "run --release --bin tools -- scenario"
 scenario-with-tracing   = "run --release --features tracing --bin tools -- scenario"
 replay-block            = "run --release --bin tools -- replay-block"
+generate-cheats-interface = "test -p foundry-cheatcodes-spec --features schema tests::"

.github/scripts/cargo-doc.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+# For EDR crates, test that docs build and they don't have warnings
+for dir in crates/edr_*/ ; do
+    if [ -d "$dir" ]; then
+      pushd "$dir" > /dev/null
+      RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features
+      popd > /dev/null
+    fi
+done
+
+# For Foundry crates, only test that docs build and allow linking to private items
+for dir in crates/foundry/* ; do
+    if [ -d "$dir" ]; then
+      pushd "$dir" > /dev/null
+      cargo doc --all-features --no-deps --document-private-items
+      popd > /dev/null
+    fi
+done
+

.github/scripts/cargo-hack-edr.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+# Execute cargo hack only for EDR crates
+for dir in crates/edr_*/ ; do
+    if [ -d "$dir" ]; then
+      pushd "$dir" > /dev/null 
+      cargo hack check --feature-powerset --exclude-no-default-features --no-dev-deps 
+      popd > /dev/null
+    fi
+done
+

.github/workflows/edr-benchmark-feat-solidity-tests.yml

@@ -0,0 +1,90 @@
+name: EDR Benchmarks for feat/solidity-tests
+
+on:
+  push:
+    branches:
+      - "feat/solidity-tests"
+  pull_request:
+    branches:
+      - "**"
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: crates/tools/js/benchmark
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  # Don't cancel in progress jobs in main
+  cancel-in-progress: ${{ github.ref != 'refs/heads/feat/solidity-tests' }}
+
+jobs:
+  benchmarks-test:
+    name: Benchmarks test
+    environment: github-action-benchmark
+    runs-on: self-hosted
+    # Only run for trusted collaborators since third-parties could run malicious code on the self-hosted benchmark runner.
+    if: github.ref == 'refs/heads/feat/solidity-tests' || github.repository == github.event.pull_request.head.repo.full_name
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+      - uses: ./.github/actions/setup-rust
+
+      - name: Install packages
+        run: pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Run benchmark tests
+        run: pnpm test
+
+  js-scenario-benchmark:
+    name: Run JS scenario runner benchmark for Hardhat Node style workload
+    environment: github-action-benchmark
+    runs-on: self-hosted
+    needs: benchmarks-test
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+      - uses: ./.github/actions/setup-rust
+
+      - name: Rust cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            target/
+          save-always: true
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Install packages
+        run: pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Run benchmark
+        run: pnpm run benchmark
+
+      - name: Validate regressions
+        run: pnpm run verify
+
+      - name: Generate report for github-action-benchmark
+        run: pnpm run --silent report | tee scenario-report.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: customSmallerIsBetter
+          output-file-path: crates/tools/js/benchmark/scenario-report.json
+          gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
+          gh-pages-branch: feat/solidity-tests
+          benchmark-data-dir-path: bench
+          github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
+          # Only save the data for main branch pushes. For PRs we only compare
+          auto-push: ${{ github.ref == 'refs/heads/feat/solidity-tests' && github.event_name != 'pull_request' }}
+          alert-threshold: "110%"
+          # Only fail on pull requests, don't break CI in main
+          fail-on-alert: ${{ github.event_name == 'pull_request' }}
+          # Enable Job Summary for PRs
+          summary-always: true
+          max-items-in-chart: 1000

.github/workflows/edr-benchmark.yml

@@ -1,9 +1,11 @@
-name: EDR Benchmark
+name: EDR Benchmarks
 
 on:
   push:
     branches:
-      - main
+      # TODO: change to `main` when we merging to main
+      # https://github.com/NomicFoundation/edr/issues/685
+      - "feat/solidity-tests"
   pull_request:
     branches:
       - "**"
@@ -16,40 +18,70 @@ defaults:
 concurrency:
   group: ${{github.workflow}}-${{github.ref}}
   # Don't cancel in progress jobs in main
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+  # TODO: change to `main` when we merging to main
+  # https://github.com/NomicFoundation/edr/issues/685
+  cancel-in-progress: ${{ github.ref != 'refs/heads/feat/solidity-tests' }}
 
 jobs:
-  js-benchmark:
-    name: Run JS scenario runner benchmark
+  benchmarks-test:
+    name: Benchmarks test
     environment: github-action-benchmark
     runs-on: self-hosted
     # Only run for trusted collaborators since third-parties could run malicious code on the self-hosted benchmark runner.
-    if: github.ref == 'refs/heads/main' || github.repository == github.event.pull_request.head.repo.full_name
+    # TODO: change ref comparison to `main` when we merging to main
+    # https://github.com/NomicFoundation/edr/issues/685
+    if: github.ref == 'refs/heads/feat/solidity-tests' || github.repository == github.event.pull_request.head.repo.full_name
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-node
       - uses: ./.github/actions/setup-rust
 
-      - name: Install package
+      - name: Install packages
         run: pnpm install --frozen-lockfile --prefer-offline
 
       - name: Run benchmark tests
         run: pnpm test
 
+  js-scenario-benchmark:
+    name: Run JS scenario runner benchmark for Hardhat Node style workload
+    environment: github-action-benchmark
+    runs-on: self-hosted
+    needs: benchmarks-test
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+      - uses: ./.github/actions/setup-rust
+
+      - name: Rust cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            target/
+          save-always: true
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Install packages
+        run: pnpm install --frozen-lockfile --prefer-offline
+
       - name: Run benchmark
-        run: pnpm run -s benchmark
+        run: pnpm run benchmark
 
       - name: Validate regressions
-        run: pnpm run -s verify
+        run: pnpm run verify
 
       - name: Generate report for github-action-benchmark
-        run: pnpm run -s report | tee report.json
+        run: pnpm run --silent report | tee scenario-report.json
 
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:
           tool: customSmallerIsBetter
-          output-file-path: crates/tools/js/benchmark/report.json
+          output-file-path: crates/tools/js/benchmark/scenario-report.json
           gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
           gh-pages-branch: main
           benchmark-data-dir-path: bench
@@ -62,3 +94,59 @@ jobs:
           # Enable Job Summary for PRs
           summary-always: true
           max-items-in-chart: 1000
+
+  js-soltests-benchmark:
+    name: Run JS Solidity test runner benchmark
+    environment: github-action-benchmark
+    runs-on: self-hosted
+    needs: benchmarks-test
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+      - uses: ./.github/actions/setup-rust
+
+      - name: Cache EDR RPC cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            **/edr-cache
+          key: edr-rs-rpc-cache-v1
+
+      - name: Rust cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            target/
+          save-always: true
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Install packages
+        run: pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Run benchmark and generate report for github-action-benchmark
+        run: pnpm run --silent soltests | tee soltest-report.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: customSmallerIsBetter
+          output-file-path: crates/tools/js/benchmark/soltest-report.json
+          gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
+          gh-pages-branch: main
+          benchmark-data-dir-path: soltests
+          github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
+          # Only save the data for main branch pushes. For PRs we only compare
+          # TODO: change to `main` when we merging to main
+          # https://github.com/NomicFoundation/edr/issues/685
+          auto-push: ${{ github.ref == 'refs/heads/feat/solidity-tests' && github.event_name != 'pull_request' }}
+          alert-threshold: "110%"
+          # Only fail on pull requests, don't break CI in main
+          fail-on-alert: ${{ github.event_name == 'pull_request' }}
+          # Enable Job Summary for PRs
+          summary-always: true
+          max-items-in-chart: 1000

.github/workflows/edr-ci.yml

@@ -11,7 +11,6 @@ on:
 
 env:
   RUSTFLAGS: -Dwarnings
-  RUSTDOCFLAGS: -Dwarnings
 
 concurrency:
   group: ${{github.workflow}}-${{github.ref}}
@@ -28,7 +27,14 @@ jobs:
       # Install pre-built binaries for cargo hack
       - uses: taiki-e/install-action@cargo-hack
 
-      - run: cargo hack check --feature-powerset --no-dev-deps
+      - name: Cargo check no default features for all crates
+        run: cargo check --all --no-default-features
+
+      - name: Cargo hack for EDR crates
+        run: |
+          chmod +x ./.github/scripts/cargo-hack-edr.sh
+          ./.github/scripts/cargo-hack-edr.sh
+        shell: bash
 
   test-edr-rs:
     name: Test EDR (${{ matrix.os }})
@@ -107,17 +113,22 @@ jobs:
       - uses: ./.github/actions/setup-rust
 
       - name: Cargo doc
-        run: cargo doc --workspace --no-deps --all-features
+        run: |
+          chmod +x ./.github/scripts/cargo-doc.sh
+          ./.github/scripts/cargo-doc.sh
+        shell: bash
 
-  lint:
-    name: Run Prettier
+  build-and-lint:
+    name: Build and lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-node
 
       - name: Install package
         run: pnpm install --frozen-lockfile --prefer-offline
+      - name: Run build script
+        run: pnpm run build:dev
       - name: Run lint script
         run: pnpm run lint
 
@@ -132,7 +143,20 @@ jobs:
         run: pnpm install --frozen-lockfile --prefer-offline
 
       - name: Build edr_napi
-        run: cd crates/edr_napi && pnpm build
+        run: cd crates/edr_napi && pnpm build:dev
 
       - name: Check that there are no uncommitted changes
         run: git diff --exit-code
+
+  edr-integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+
+      - name: Install package
+        run: pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Run integration tests
+        run: pnpm run --recursive --filter './js/integration-tests/*' test