update integration

.github/workflows/ci.yml

@@ -88,9 +88,30 @@ jobs:
           pattern: build-${{ matrix.os }}
           merge-multiple: true
       - run: |
-          cp artifacts/libec_go_lib.* ecgo/compile/wrapper/
+          cp artifacts/libec_go_lib.* ecgo/rust/wrapper/
           cd ecgo
           go test ./test/
+  
+  test-go-keccak-serve:
+    runs-on: 7950x3d
+    needs: build-rust
+    steps:
+      - uses: styfle/[email protected]
+      - uses: actions/checkout@v4
+      - name: Setup Go 1.21.x
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ./artifacts
+          pattern: build-ubuntu-latest
+          merge-multiple: true
+      - run: |
+          cp artifacts/libec_go_lib.* ecgo/rust/wrapper/
+          cd ecgo
+          go run examples/keccak_full/main.go
 
   lint:
     runs-on: macos-latest

build-rust.sh

@@ -3,4 +3,4 @@ cd "$(dirname "$0")"
 cd expander_compiler/ec_go_lib
 cargo build --release
 cd ..
-cp target/release/libec_go_lib.so ../ecgo/compile/wrapper/
+cp target/release/libec_go_lib.so ../ecgo/rust/wrapper/

ecgo/api.go

@@ -9,10 +9,10 @@ import (
 	"reflect"
 
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/builder"
-	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/compile"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/irsource"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/irwg"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/layered"
+	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/rust"
 	"github.com/consensys/gnark/frontend"
 	"github.com/consensys/gnark/frontend/schema"
 	"github.com/consensys/gnark/logger"
@@ -82,7 +82,7 @@ func Compile(field *big.Int, circuit frontend.Circuit, opts ...frontend.CompileO
 	rc := root.Finalize()
 	_ = rc
 	//os.WriteFile("p1.txt", irsource.SerializeRootCircuit(rc), 0644)
-	irwg, lc, err := compile.Compile(rc)
+	irwg, lc, err := rust.Compile(rc)
 	if err != nil {
 		return nil, err
 	}

ecgo/integration/expander_exec.go

@@ -8,32 +8,22 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
-	"os/exec"
 	"runtime"
 
-	"github.com/juju/fslock"
+	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/rust"
 )
 
-//go:embed bin/expander-exec-linux-avx2
-var embed_expander_exec_linux_avx2 []byte
-
-//go:embed bin/expander-exec-macos
-var embed_expander_exec_macos []byte
-
 type Prover struct {
 	circuitDir string
 	filename   string
 }
 
-const bin_loc = "./__expander-exec"
-const bin_lock_loc = "./__expander-exec.lock"
-
 func is_url(s string) bool {
 	_, err := url.ParseRequestURI(s)
 	return err == nil
 }
 
+// for compatibility with keccak_serve
 func GetIntegrationBinLoc() string {
 	switch runtime.GOOS {
 	case "darwin":
@@ -48,33 +38,6 @@ func GetIntegrationBinLoc() string {
 
 // format from external specification, maxConcurrency and releaseFlags are not used now
 func NewProver(circuitDir string, filename string, maxConcurrency int, releaseFlag bool) (*Prover, error) {
-	if is_url(filename) {
-		return &Prover{
-			circuitDir: circuitDir,
-			filename:   filename,
-		}, nil
-	}
-	lock := fslock.New(bin_lock_loc)
-	lock.Lock()
-	if _, err := os.Stat(bin_loc); os.IsNotExist(err) {
-		// write the binary to the file
-		switch runtime.GOOS {
-		case "darwin":
-			err := os.WriteFile(bin_loc, embed_expander_exec_macos, 0700)
-			if err != nil {
-				panic("could not write file")
-			}
-		case "linux":
-			err := os.WriteFile(bin_loc, embed_expander_exec_linux_avx2, 0700)
-			if err != nil {
-				panic("could not write file")
-			}
-		default:
-			println("Unsupported OS: ", runtime.GOOS)
-			panic("Unsupported OS")
-		}
-	}
-	lock.Unlock()
 	return &Prover{
 		circuitDir: circuitDir,
 		filename:   filename,
@@ -102,42 +65,7 @@ func (p *Prover) Prove(witnessData []byte) ([]byte, error) {
 		}
 		return proof_and_claim, nil
 	}
-	// use external prover executable
-	// format: ./bin/expander-exec prove <input:circuit_file> <input:witness_file> <output:proof>
-	outputFile, err := os.CreateTemp("", "output")
-	if err != nil {
-		fmt.Println("Error creating temporary file:", err)
-		return nil, err
-	}
-
-	tmpFile, err := os.CreateTemp("", "witness")
-	if err != nil {
-		fmt.Println("Error creating temporary file:", err)
-		return nil, err
-	}
-
-	if _, err := tmpFile.Write(witnessData); err != nil {
-		fmt.Println("Error writing to temporary file:", err)
-		return nil, err
-	}
-
-	cmd := exec.Command(bin_loc, "prove", p.circuitDir, tmpFile.Name(), outputFile.Name())
-	// println("cmd: ", cmd.String())
-	if err := cmd.Run(); err != nil {
-		fmt.Println("Error running command:", err)
-		return nil, err
-	}
-	defer os.Remove(tmpFile.Name()) // Clean up the file when done
-
-	proof_and_claim, err := os.ReadFile(outputFile.Name())
-	if err != nil {
-		fmt.Println("Error reading output file:", err)
-		return nil, err
-	}
-	os.Remove(outputFile.Name())
-	defer os.Remove(outputFile.Name()) // Clean up the file when done
-
-	return proof_and_claim, nil
+	return rust.ProveFile(p.circuitDir, witnessData), nil
 }
 
 func (p *Prover) Verify(witnessData []byte, proof_and_claim []byte) (bool, error) {
@@ -166,39 +94,5 @@ func (p *Prover) Verify(witnessData []byte, proof_and_claim []byte) (bool, error
 		}
 		return string(verificationResult) == "success", nil
 	}
-	// use external prover executable
-	// format: ./bin/expander-exec verify <input:circuit_file> <input:witness_file> <input:proof>
-
-	tmpFileWit, err := os.CreateTemp("", "witness")
-	if err != nil {
-		fmt.Println("Error creating temporary file:", err)
-		return false, err
-	}
-
-	if _, err := tmpFileWit.Write(witnessData); err != nil {
-		fmt.Println("Error writing to temporary file:", err)
-		return false, err
-	}
-
-	tmpFileProof, err := os.CreateTemp("", "proof")
-	if err != nil {
-		fmt.Println("Error creating temporary file:", err)
-		return false, err
-	}
-
-	if _, err := tmpFileProof.Write(proof_and_claim); err != nil {
-		fmt.Println("Error writing to temporary file:", err)
-		return false, err
-	}
-
-	cmd := exec.Command(bin_loc, "verify", p.circuitDir, tmpFileWit.Name(), tmpFileProof.Name())
-	// println("cmd: ", cmd.String())
-	if err := cmd.Run(); err != nil {
-		// no need to print as it is expected to fail on invalid proof
-		return false, err
-	}
-	defer os.Remove(tmpFileWit.Name())   // Clean up the file when done
-	defer os.Remove(tmpFileProof.Name()) // Clean up the file when done
-
-	return true, nil
+	return rust.VerifyFile(p.circuitDir, witnessData, proof_and_claim), nil
 }

ecgo/layered/serialize_old.go

@@ -2,7 +2,9 @@ package layered
 
 import (
 	"math/big"
+	"os"
 
+	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/field"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/utils"
 )
 
@@ -184,3 +186,23 @@ func DeserializeRootCircuit(buf []byte) *RootCircuit {
 	rc.ExpectedNumOutputZeroes = int(rc.Circuits[rc.Layers[len(rc.Layers)-1]].OutputLen)
 	return rc
 }
+
+func DetectFieldIdFromFile(fn string) uint64 {
+	// Read the first 4 bytes of the file
+	file, err := os.Open(fn)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+	buf := make([]byte, 40)
+	n, err := file.Read(buf)
+	if err != nil || n != 40 {
+		panic(err)
+	}
+	in := utils.NewInputBuf(buf)
+	if in.ReadUint64() != 3770719418566461763 {
+		panic("invalid file header")
+	}
+	f := in.ReadBigInt(32)
+	return field.GetFieldId(field.GetFieldFromOrder(f))
+}

ecgo/compile/compile.go → ecgo/rust/rust.go

@@ -1,11 +1,11 @@
-package compile
+package rust
 
 import (
-	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/compile/wrapper"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/field"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/irsource"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/irwg"
 	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/layered"
+	"github.com/PolyhedraZK/ExpanderCompilerCollection/ecgo/rust/wrapper"
 )
 
 func Compile(rc *irsource.RootCircuit) (*irwg.RootCircuit, *layered.RootCircuit, error) {
@@ -18,3 +18,11 @@ func Compile(rc *irsource.RootCircuit) (*irwg.RootCircuit, *layered.RootCircuit,
 	lc := layered.DeserializeNewCompilerRootCircuit(lcSer)
 	return irWg, lc, nil
 }
+
+func ProveFile(circuitFilename string, witnessBytes []byte) []byte {
+	return wrapper.ProveCircuitFile(circuitFilename, witnessBytes, layered.DetectFieldIdFromFile(circuitFilename))
+}
+
+func VerifyFile(circuitFilename string, witnessBytes []byte, proofBytes []byte) bool {
+	return wrapper.VerifyCircuitFile(circuitFilename, witnessBytes, proofBytes, layered.DetectFieldIdFromFile(circuitFilename))
+}

ecgo/compile/wrapper/wrapper.go → ecgo/rust/wrapper/wrapper.go

@@ -22,7 +22,7 @@ import (
 	"github.com/consensys/gnark/logger"
 )
 
-const ABI_VERSION = 2
+const ABI_VERSION = 3
 
 func currentFileDirectory() string {
 	_, fileName, _, ok := runtime.Caller(1)
@@ -34,6 +34,8 @@ func currentFileDirectory() string {
 }
 
 var compilePtr unsafe.Pointer = nil
+var proveCircuitFilePtr unsafe.Pointer = nil
+var verifyCircuitFilePtr unsafe.Pointer = nil
 var compilePtrLock sync.Mutex
 
 func downloadFile(url string, filepath string) error {
@@ -163,10 +165,6 @@ func initCompilePtr() {
 	if handle == nil {
 		panic("failed to load libec_go_lib")
 	}
-	compilePtr = C.dlsym(handle, C.CString("compile"))
-	if compilePtr == nil {
-		panic("failed to load compile function")
-	}
 	abiVersionPtr := C.dlsym(handle, C.CString("abi_version"))
 	if abiVersionPtr == nil {
 		panic("failed to load abi_version function")
@@ -175,6 +173,18 @@ func initCompilePtr() {
 	if abiVersion != ABI_VERSION {
 		panic("abi_version mismatch, please consider update the go package")
 	}
+	compilePtr = C.dlsym(handle, C.CString("compile"))
+	if compilePtr == nil {
+		panic("failed to load compile function")
+	}
+	proveCircuitFilePtr = C.dlsym(handle, C.CString("prove_circuit_file"))
+	if proveCircuitFilePtr == nil {
+		panic("failed to load prove_circuit_file function")
+	}
+	verifyCircuitFilePtr = C.dlsym(handle, C.CString("verify_circuit_file"))
+	if verifyCircuitFilePtr == nil {
+		panic("failed to load verify_circuit_file function")
+	}
 }
 
 func CompileWithRustLib(s []byte, configId uint64) ([]byte, []byte, error) {
@@ -199,3 +209,27 @@ func CompileWithRustLib(s []byte, configId uint64) ([]byte, []byte, error) {
 
 	return irWitnessGen, layered, nil
 }
+
+func ProveCircuitFile(circuitFilename string, witness []byte, configId uint64) []byte {
+	initCompilePtr()
+	bytesFn := []byte(circuitFilename)
+	cf := C.ByteArray{data: (*C.uint8_t)(C.CBytes(bytesFn)), length: C.uint64_t(len(bytesFn))}
+	defer C.free(unsafe.Pointer(cf.data))
+	wi := C.ByteArray{data: (*C.uint8_t)(C.CBytes(witness)), length: C.uint64_t(len(witness))}
+	defer C.free(unsafe.Pointer(wi.data))
+	proof := C.prove_circuit_file(proveCircuitFilePtr, cf, wi, C.uint64_t(configId))
+	defer C.free(unsafe.Pointer(proof.data))
+	return C.GoBytes(unsafe.Pointer(proof.data), C.int(proof.length))
+}
+
+func VerifyCircuitFile(circuitFilename string, witness []byte, proof []byte, configId uint64) bool {
+	initCompilePtr()
+	bytesFn := []byte(circuitFilename)
+	cf := C.ByteArray{data: (*C.uint8_t)(C.CBytes(bytesFn)), length: C.uint64_t(len(bytesFn))}
+	defer C.free(unsafe.Pointer(cf.data))
+	wi := C.ByteArray{data: (*C.uint8_t)(C.CBytes(witness)), length: C.uint64_t(len(witness))}
+	defer C.free(unsafe.Pointer(wi.data))
+	pr := C.ByteArray{data: (*C.uint8_t)(C.CBytes(proof)), length: C.uint64_t(len(proof))}
+	defer C.free(unsafe.Pointer(pr.data))
+	return C.verify_circuit_file(verifyCircuitFilePtr, cf, wi, pr, C.uint64_t(configId)) != 0
+}