feat: go-corset front end integration

.github/actions/setup-go-corset/action.yml

@@ -12,4 +12,4 @@ runs:
 
     - name: Install Go Corset
       shell: bash
-      run: go install github.com/consensys/go-corset/cmd/go-corset@5cb0c28
+      run: go install github.com/consensys/go-corset/cmd/go-corset@f89cf145

.github/actions/setup-rust-corset/action.yml

@@ -9,7 +9,7 @@ inputs:
     # Corresponds to the corset release tag for the desired version.
     description: 'Corset tag to use'
     required: false
-    default: 'v9.7.17'
+    default: 'v9.7.18'
   asset_id:
     # Corresponds to the Github asset id for the asset published as part
     # of the release.  Specifically, it should be the id for the asset

.github/workflows/gradle-ethereum-tests.yml

@@ -72,14 +72,15 @@ jobs:
         run: ./gradlew generateGeneralStateReferenceTests -Dorg.gradle.caching=true
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: fields,expand,expand,expand
+          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
 
       - name: Run General State Reference Tests
         run: GOMEMLIMIT=64GiB ./gradlew -Dblockchain=Ethereum referenceGeneralStateTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --air
+          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
 
       - name: Upload test report
         if: always()

.github/workflows/gradle-nightly-tests.yml

@@ -24,11 +24,12 @@ jobs:
           go-corset: true
 
       - name: Run Nightly tests
-        run: GOMEMLIMIT=96GiB ./gradlew nightlyTests
+        run: GOMEMLIMIT=64GiB ./gradlew nightlyTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --mir
+          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Upload test report
         if: ${{ always() }}

.github/workflows/gradle-tests.yml

@@ -105,8 +105,9 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -b1024 -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -b1024 -vw --ansi-escapes=false --report --air
           UNIT_TESTS_PARALLELISM: 1
+          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Upload test report
         if: ${{ always() }}
@@ -168,7 +169,8 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -b1024 -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -b1024 -vw --ansi-escapes=false --report --air
+          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Upload test report
         if: ${{ always() }}

.github/workflows/gradle-weekly-tests.yml

@@ -67,7 +67,8 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --air
+          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Failure Notification
         if: ${{ failure() || cancelled() }}

.github/workflows/reference-blockchain-tests.yml

@@ -39,20 +39,16 @@ jobs:
         id: extract_branch
         run: |
             echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
-      - name: Set up JDK 21
-        uses: actions/setup-java@v3
-        with:
-          java-version: 21
-          distribution: temurin
 
-      - name: Setup Go Corset
-        uses: ./.github/actions/setup-go-corset
+      - name: Setup Test Environment
+        uses: ./.github/actions/setup-environment
+        with:
+          go-corset: true
 
       - name: Generate block chain reference tests
         run: ./gradlew :reference-tests:generateBlockchainReferenceTests -Dorg.gradle.caching=true
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: fields,expand,expand,expand
 
       - name: Download artifact
         uses: dawidd6/action-download-artifact@v6
@@ -75,7 +71,8 @@ jobs:
           REFERENCE_TESTS_PARALLELISM: 2
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -wd --ansi-escapes=false --report --air
+          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --air
+          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
           FAILED_TEST_JSON_DIRECTORY: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/
           FAILED_MODULE: ${{ inputs.failed_module || '' }}
           FAILED_CONSTRAINT: ${{ inputs.failed_constraint || '' }}

arithmetization/build.gradle

@@ -25,7 +25,7 @@ version = rootProject.version
 
 apply from: rootProject.file("gradle/java.gradle")
 apply from: rootProject.file("gradle/dependency-management.gradle")
-apply from: rootProject.file('gradle/common-dependencies.gradle')
+apply from: rootProject.file("gradle/common-dependencies.gradle")
 apply from: rootProject.file("gradle/corset.gradle")
 apply from: rootProject.file("gradle/tests.gradle")
 apply from: rootProject.file("gradle/build-aliases.gradle")

gradle/corset.gradle

@@ -19,38 +19,54 @@ tasks.register('corsetExists') {
             commandLine "corset", "-V"
         }
         if (result.getExitValue() != 0){
-            throw new GradleException('Corset not found, skipping corsetTests')
+            throw new GradleException('Corset not found!')
         }
     }
 }
 
+// Allowing overriding the default zkevm.bin file.
+def ZKEVM_BIN = System.getenv("ZKEVM_BIN") ?: "zkevm.bin"
+
 tasks.register('buildZkevmBin', Exec) {
     workingDir "${project.rootDir}/linea-constraints/"
-    commandLine 'make', 'zkevm.bin'
-    // sanity check target chain
-    def chain = System.getProperty('blockchain')
-    // note: null indicates the default (which is Linea).
-    if(chain != null && chain != "Linea") {
-        println("*** incorrect target chain (expected 'Linea' was '" + chain + "')\n\n")
-        assert false
+    // Sanity check executing this makes sense.
+    doFirst {    
+        // Record zkevm.bin for CorsetValidator
+        environment.put("ZKEVM_BIN", ZKEVM_BIN)
+        // sanity check target chain
+        def chain = System.getProperty('blockchain')
+        // note: null indicates the default (which is Linea).
+        if(chain != null && chain != "Linea") {
+            println("*** incorrect target chain (expected 'Linea' was '" + chain + "')\n\n")
+            assert false
+        }
     }
+    //
+    commandLine 'make', ZKEVM_BIN        
     // store the output instead of printing to the console
     standardOutput = new ByteArrayOutputStream()
     ext.output = {
         return standardOutput.toString()
     }
 }
 
+// Allowing overriding the default zkevm.bin file.
+def ZKEVM_REFTESTS_BIN = System.getenv("ZKEVM_BIN") ?: "zkevm_for_reference_tests.bin"
+
 tasks.register('buildReferenceTestsZkevmBin', Exec) {
     workingDir "${project.rootDir}/linea-constraints/"
-    commandLine 'make', 'zkevm_for_reference_tests.bin'
-    // sanity check target chain
-    def chain = System.getProperty('blockchain')
+    // Sanity check executing this makes sense.
+    doFirst {
+        // sanity check target chain
+        def chain = System.getProperty('blockchain')
+        //
+        if(chain != "Ethereum") {
+            println("*** incorrect target chain (expected 'Ethereum' was '" + chain + "')\n\n")
+            assert false
+        }
+    }
     //
-    if(chain != "Ethereum") {
-        println("*** incorrect target chain (expected 'Ethereum' was '" + chain + "')\n\n")
-        assert false
-    }    
+    commandLine 'make', ZKEVM_REFTESTS_BIN    
     // Store the output instead of printing to the console
     standardOutput = new ByteArrayOutputStream()
     ext.output = {

gradle/tests.gradle

@@ -41,8 +41,6 @@ apply plugin: 'com.adarshr.test-logger'
   * - 'root.log.level' and 'evm.log.level': allow to control the log level used during the tests.
   */
 tasks.withType(Test).configureEach {
-    dependsOn 'buildZkevmBin'
-
   minHeapSize = "4g"
   maxHeapSize = "8g"
 
@@ -118,6 +116,8 @@ tasks.withType(Test).configureEach {
 }
 
 tasks.test.configure {
+  dependsOn(buildZkevmBin)
+
   systemProperty("junit.jupiter.execution.parallel.enabled", true)
   systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
   systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent")
@@ -133,6 +133,8 @@ tasks.test.configure {
 }
 
 tasks.register("fastReplayTests", Test) {
+  dependsOn(buildZkevmBin)
+
   if (System.getenv().containsKey("REPLAY_TESTS_PARALLELISM")) {
     systemProperty("junit.jupiter.execution.parallel.enabled", true)
     systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
@@ -149,6 +151,8 @@ tasks.register("fastReplayTests", Test) {
 }
 
 tasks.register("nightlyReplayTests", Test) {
+  dependsOn(buildZkevmBin)
+
   boolean isCiServer = System.getenv().containsKey("CI")
   maxHeapSize = isCiServer ? "32g" : "8g"
 
@@ -168,6 +172,8 @@ tasks.register("nightlyReplayTests", Test) {
 }
 
 tasks.register("nightlyTests", Test) {
+  dependsOn(buildZkevmBin)
+
   boolean isCiServer = System.getenv().containsKey("CI")
   maxHeapSize = isCiServer ? "32g" : "8g"
 
@@ -184,6 +190,8 @@ tasks.register("nightlyTests", Test) {
 }
 
 tasks.register("weeklyTests", Test) {
+  dependsOn(buildZkevmBin)
+
   systemProperty("junit.jupiter.execution.parallel.enabled", true)
   systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
   systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent")

gradle/trace-files.gradle

@@ -108,7 +108,7 @@ tasks.register('binreftable', TraceFilesTask) {
 ['txndata'].each {moduleName ->
     tasks.register(moduleName, TraceFilesTask) {
         group "Trace files generation"
-         dependsOn corsetExists
+        dependsOn corsetExists
         module = moduleName
         files = ["${moduleName}/columns.lisp", "constants/constants.lisp", "${moduleName}/constants.lisp"]
     }

reference-tests/build.gradle

@@ -19,7 +19,6 @@ plugins {
 
 apply from: rootProject.file("gradle/corset.gradle")
 apply from: rootProject.file("gradle/java.gradle")
-apply from: rootProject.file("gradle/trace-files.gradle")
 apply from: rootProject.file("gradle/dependency-management.gradle")
 apply from: rootProject.file('gradle/common-dependencies.gradle')
 apply from: rootProject.file("gradle/lint.gradle")
@@ -55,6 +54,7 @@ tasks.register("generateGeneralStateReferenceTests", RefTestGenerationTask) {
 }
 
 tasks.register('referenceBlockchainTests', Test) {
+  dependsOn(buildReferenceTestsZkevmBin)
   description = 'Runs ETH reference blockchain tests.'
   dependsOn generateBlockchainReferenceTests
   environment.put("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "BlockchainReferenceTestOutcome.json")
@@ -69,9 +69,10 @@ tasks.register('referenceBlockchainTests', Test) {
 }
 
 tasks.register('referenceGeneralStateTests', Test) {
+  dependsOn(buildReferenceTestsZkevmBin)
   description = 'Runs ETH reference general state tests.'
   dependsOn generateGeneralStateReferenceTests
-
+  // Record zkevm.bin for CorsetValidator
   environment.put("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "GeneralStateReferenceTestOutcome.json")
 
   useJUnitPlatform {
@@ -88,9 +89,9 @@ tasks.register('referenceGeneralStateTests', Test) {
   tasks.named("referenceBlockchainTests"),
 ].forEach {
   it.configure {
-    dependsOn buildReferenceTestsZkevmBin
-    environment.put("ZKEVM_BIN", "zkevm_for_reference_tests.bin")
-
+    // Record zkevm.bin for CorsetValidator
+    environment.put("ZKEVM_BIN", System.getenv("ZKEVM_BIN") ?: "zkevm_for_reference_tests.bin")
+    // Configure parallelism
     systemProperty("junit.jupiter.execution.timeout.default", "15 m")  // 5 minutes
     systemProperty("junit.jupiter.execution.parallel.enabled", "true")
     systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")