Merge branch 'zz/circuit_names' into zz/trivial_layers

.github/workflows/ci.yml

@@ -18,17 +18,19 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "expander_compiler -> expander_compiler/target"
+          # The prefix cache key, this can be changed to start a new cache manually.
+          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
       - if: matrix.os == 'macos-latest'
         run: brew install openmpi
       - if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install libopenmpi-dev -y
       - name: Build
-        run: cargo build --release --manifest-path=expander_compiler/ec_go_lib/Cargo.toml
+        run: cargo build --release
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: build-${{ matrix.os }}
-          path: expander_compiler/target/release/libec_go_lib.*
+          path: target/release/libec_go_lib.*
 
   upload-rust:
     needs: [build-rust, test-rust, test-rust-avx512, lint]
@@ -68,11 +70,13 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "expander_compiler -> expander_compiler/target"
+          # The prefix cache key, this can be changed to start a new cache manually.
+          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
       - if: matrix.os == 'macos-latest'
         run: brew install openmpi
       - if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install libopenmpi-dev -y
-      - run: cargo test --manifest-path=expander_compiler/Cargo.toml 
+      - run: cargo test
 
   test-rust-avx512:
     runs-on: 7950x3d
@@ -83,7 +87,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "expander_compiler -> expander_compiler/target"
-      - run: RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo test --manifest-path=expander_compiler/Cargo.toml
+          # The prefix cache key, this can be changed to start a new cache manually.
+          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
+      - run: RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo test
 
   test-go:
     runs-on: ${{ matrix.os }}
@@ -109,7 +115,8 @@ jobs:
       - if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install libopenmpi-dev -y
       - run: |
-          cp artifacts/libec_go_lib.* ecgo/rust/wrapper/
+          mkdir -p ~/.cache/ExpanderCompilerCollection
+          cp artifacts/libec_go_lib.* ~/.cache/ExpanderCompilerCollection
           cd ecgo
           go test ./test/
   
@@ -131,7 +138,8 @@ jobs:
           merge-multiple: true
       - run: |
           sudo apt-get update && sudo apt-get install libopenmpi-dev -y
-          cp artifacts/libec_go_lib.* ecgo/rust/wrapper/
+          mkdir -p ~/.cache/ExpanderCompilerCollection
+          cp artifacts/libec_go_lib.* ~/.cache/ExpanderCompilerCollection
           cd ecgo
           go run examples/keccak_full/main.go
 
@@ -143,5 +151,5 @@ jobs:
         with:
           components: rustfmt, clippy
       - run: brew install openmpi
-      - run: cargo fmt --all --manifest-path=expander_compiler/Cargo.toml -- --check
-      - run: cargo clippy --manifest-path=expander_compiler/Cargo.toml
+      - run: cargo fmt --all -- --check
+      - run: cargo clippy

Cargo.toml

@@ -0,0 +1,29 @@
+[workspace]
+resolver = "2"
+members = ["expander_compiler", "expander_compiler/ec_go_lib"]
+
+[profile.test]
+opt-level = 3
+
+[profile.dev]
+opt-level = 3
+
+
+[workspace.dependencies]
+ark-std = "0.4.0"
+rand = "0.8.5"
+chrono = "0.4"
+clap = { version = "4.1", features = ["derive"] }
+ethnum = "1.5.0"
+tiny-keccak = { version = "2.0", features = ["keccak"] }
+halo2curves = { git = "https://github.com/PolyhedraZK/halo2curves", default-features = false, features = [
+    "bits",
+] }
+arith = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
+expander_config = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly", package = "config" }
+expander_circuit = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly", package = "circuit" }
+gkr = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
+gf2 = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
+mersenne31 = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
+expander_transcript = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" , package = "transcript" }
+

build-rust-avx512.sh

@@ -1,6 +1,5 @@
 #!/bin/sh
 cd "$(dirname "$0")"
-cd expander_compiler/ec_go_lib
-RUSTFLAGS="-C target-cpu=native -C target-features=+avx512f" cargo build --release
-cd ..
-cp target/release/libec_go_lib.so ../ecgo/rust/wrapper/
+RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo build --release
+mkdir -p ~/.cache/ExpanderCompilerCollection
+cp target/release/libec_go_lib.so ~/.cache/ExpanderCompilerCollection

build-rust.sh

@@ -1,6 +1,5 @@
 #!/bin/sh
 cd "$(dirname "$0")"
-cd expander_compiler/ec_go_lib
 cargo build --release
-cd ..
-cp target/release/libec_go_lib.so ../ecgo/rust/wrapper/
+mkdir -p ~/.cache/ExpanderCompilerCollection
+cp target/release/libec_go_lib.so ~/.cache/ExpanderCompilerCollection

ecgo/builder/root.go

@@ -44,10 +44,7 @@ func (r *Root) PublicVariable(f schema.LeafInfo) frontend.Variable {
 		ExtraId: 2 + uint64(r.nbPublicInputs),
 	})
 	r.nbPublicInputs++
-	// Currently, new version of public input is not support by Expander
-	// So we use a hint to isolate it in witness solver
-	x, _ := r.NewHint(IdentityHint, 1, r.addVar())
-	return x[0]
+	return r.addVar()
 }
 
 // SecretVariable creates a new secret variable for the circuit.

ecgo/examples/poseidon_m31/main.go

@@ -72,7 +72,7 @@ func M31CircuitBuild() {
 
 	layered_circuit := circuit.GetLayeredCircuit()
 	// circuit.GetCircuitIr().Print()
-	err = os.WriteFile("circuit.txt", layered_circuit.Serialize(), 0o644)
+	err = os.WriteFile("poseidon_120_circuit_m31.txt", layered_circuit.Serialize(), 0o644)
 	if err != nil {
 		panic(err)
 	}
@@ -81,7 +81,7 @@ func M31CircuitBuild() {
 	if err != nil {
 		panic(err)
 	}
-	err = os.WriteFile("witness.txt", witness.Serialize(), 0o644)
+	err = os.WriteFile("poseidon_120_witness_m31.txt", witness.Serialize(), 0o644)
 	if err != nil {
 		panic(err)
 	}

ecgo/rust/wrapper/wrapper.go

@@ -24,13 +24,14 @@ import (
 
 const ABI_VERSION = 4
 
-func currentFileDirectory() string {
-	_, fileName, _, ok := runtime.Caller(1)
-	if !ok {
-		panic("can't get current file directory")
+func getCacheDir() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
 	}
-	dir, _ := filepath.Split(fileName)
-	return dir
+	cacheDir := filepath.Join(homeDir, ".cache", "ExpanderCompilerCollection")
+	err = os.MkdirAll(cacheDir, 0755)
+	return cacheDir, err
 }
 
 var compilePtr unsafe.Pointer = nil
@@ -158,8 +159,11 @@ func initCompilePtr() {
 	if compilePtr != nil {
 		return
 	}
-	curDir := currentFileDirectory()
-	soPath := filepath.Join(curDir, getLibName())
+	cacheDir, err := getCacheDir()
+	if err != nil {
+		panic(fmt.Sprintf("failed to get cache dir: %v", err))
+	}
+	soPath := filepath.Join(cacheDir, getLibName())
 	updateLib(soPath)
 	handle := C.dlopen(C.CString(soPath), C.RTLD_LAZY)
 	if handle == nil {

expander_compiler/Cargo.toml

@@ -3,11 +3,6 @@ name = "expander_compiler"
 version = "0.1.0"
 edition = "2021"
 
-[profile.test]
-opt-level = 3
-
-[profile.dev]
-opt-level = 3
 
 [dependencies]
 ark-std.workspace = true
@@ -24,30 +19,6 @@ arith.workspace = true
 gf2.workspace = true
 mersenne31.workspace = true
 
-[workspace]
-members = [
-    "ec_go_lib"
-]
-
-[workspace.dependencies]
-ark-std = "0.4.0"
-rand = "0.8.5"
-chrono = "0.4"
-clap = { version = "4.1", features = ["derive"] }
-ethnum = "1.5.0"
-tiny-keccak = { version = "2.0", features = ["keccak"] }
-halo2curves = { git = "https://github.com/PolyhedraZK/halo2curves", default-features = false, features = [
-    "bits",
-] }
-arith = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
-expander_config = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly", package = "config" }
-expander_circuit = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly", package = "circuit" }
-gkr = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
-gf2 = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
-mersenne31 = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" }
-expander_transcript = { git = "https://github.com/PolyhedraZK/Expander", branch = "nightly" , package = "transcript" }
-
-
 [[bin]]
 name = "trivial_circuit"
-path = "bin/trivial_circuit.rs"
+path = "bin/trivial_circuit.rs"

expander_compiler/ec_go_lib/src/lib.rs

@@ -29,26 +29,18 @@ fn compile_inner_with_config<C>(ir_source: Vec<u8>) -> Result<(Vec<u8>, Vec<u8>)
 where
     C: config::Config,
 {
-    let ir_source =
-        ir::source::RootCircuit::<C>::deserialize_from(&ir_source[..]).map_err(|e| {
-            format!(
-                "failed to deserialize the source circuit: {}",
-                e.to_string()
-            )
-        })?;
+    let ir_source = ir::source::RootCircuit::<C>::deserialize_from(&ir_source[..])
+        .map_err(|e| format!("failed to deserialize the source circuit: {}", e))?;
     let (ir_witness_gen, layered) =
         expander_compiler::compile::compile(&ir_source).map_err(|e| e.to_string())?;
     let mut ir_wg_s: Vec<u8> = Vec::new();
-    ir_witness_gen.serialize_into(&mut ir_wg_s).map_err(|e| {
-        format!(
-            "failed to serialize the witness generator: {}",
-            e.to_string()
-        )
-    })?;
+    ir_witness_gen
+        .serialize_into(&mut ir_wg_s)
+        .map_err(|e| format!("failed to serialize the witness generator: {}", e))?;
     let mut layered_s: Vec<u8> = Vec::new();
     layered
         .serialize_into(&mut layered_s)
-        .map_err(|e| format!("failed to serialize the layered circuit: {}", e.to_string()))?;
+        .map_err(|e| format!("failed to serialize the layered circuit: {}", e))?;
     Ok((ir_wg_s, layered_s))
 }
 
@@ -170,7 +162,7 @@ where
         expander_config::MPIConfig::new(),
     );
     let mut circuit = expander_circuit::Circuit::<C>::load_circuit(circuit_filename);
-    let witness = layered::witness::Witness::<CC>::deserialize_from(&witness[..]).unwrap();
+    let witness = layered::witness::Witness::<CC>::deserialize_from(witness).unwrap();
     let (simd_input, simd_public_input) = witness.to_simd::<C::SimdCircuitField>();
     circuit.layers[0].input_vals = simd_input;
     circuit.public_input = simd_public_input;
@@ -194,7 +186,7 @@ where
         expander_config::MPIConfig::new(),
     );
     let mut circuit = expander_circuit::Circuit::<C>::load_circuit(circuit_filename);
-    let witness = layered::witness::Witness::<CC>::deserialize_from(&witness[..]).unwrap();
+    let witness = layered::witness::Witness::<CC>::deserialize_from(witness).unwrap();
     let (simd_input, simd_public_input) = witness.to_simd::<C::SimdCircuitField>();
     circuit.layers[0].input_vals = simd_input;
     circuit.public_input = simd_public_input.clone();