fix final build and update gf2 example

ecgo/examples/keccak_gf2/main.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-const NHashes = 1
+const NHashes = 8
 
 const CheckBits = 256
 
@@ -203,12 +203,12 @@ func and(api frontend.API, a []frontend.Variable, b []frontend.Variable) []front
 	nbits := len(a)
 	bitsRes := make([]frontend.Variable, nbits)
 	for i := 0; i < nbits; i++ {
-		x := api.(ecgo.API).ToSingleVariable(a[i])
-		y := api.(ecgo.API).ToSingleVariable(b[i])
+		//x := api.(ecgo.API).ToSingleVariable(a[i])
+		//y := api.(ecgo.API).ToSingleVariable(b[i])
 		//fmt.Println(api.(ecgo.API).LayerOf(x))
-		bitsRes[i] = api.Mul(x, y)
+		//bitsRes[i] = api.Mul(x, y)
 		//fmt.Println(bitsRes[i])
-		//bitsRes[i] = api.Mul(a[i], b[i])
+		bitsRes[i] = api.Mul(a[i], b[i])
 		//bitsRes[i] = api.(ecgo.API).ToSingleVariable(bitsRes[i])
 		//fmt.Println(bitsRes[i])
 	}
@@ -247,7 +247,7 @@ func copyOutUnaligned(api frontend.API, s [][]frontend.Variable, rate, outputLen
 }
 
 type keccak256Circuit struct {
-	P   [NHashes][136 * 8]frontend.Variable
+	P   [NHashes][64 * 8]frontend.Variable
 	Out [NHashes][CheckBits]frontend.Variable
 }
 
@@ -259,11 +259,21 @@ func checkKeccak(api frontend.API, P, Out []frontend.Variable) {
 			ss[i][j] = 0
 		}
 	}
+	newP := make([]frontend.Variable, 64*8)
+	copy(newP, P)
+	appendData := make([]byte, 136-64)
+	appendData[0] = 1
+	appendData[135-64] = 0x80
+	for i := 0; i < 136-64; i++ {
+		for j := 0; j < 8; j++ {
+			newP = append(newP, int((appendData[i]>>j)&1))
+		}
+	}
 	p := make([][]frontend.Variable, 17)
 	for i := 0; i < 17; i++ {
 		p[i] = make([]frontend.Variable, 64)
 		for j := 0; j < 64; j++ {
-			p[i][j] = P[i*64+j]
+			p[i][j] = newP[i*64+j]
 		}
 	}
 	ss = xorIn(api, ss, p)
@@ -294,18 +304,14 @@ func main() {
 	os.WriteFile("circuit.txt", c.Serialize(), 0o644)
 
 	for k := 0; k < NHashes; k++ {
-		for i := 0; i < 136*8; i++ {
+		for i := 0; i < 64*8; i++ {
 			circuit.P[k][i] = 0
 		}
 
-		length := rand.Intn(130 + 2)
-		data := make([]byte, length)
+		data := make([]byte, 64)
 		rand.Read(data)
 		hash := crypto.Keccak256Hash(data)
-		data = append(data, 1)
-		data = append(data, make([]byte, 200)...)
-		data[135] = 0x80
-		for i := 0; i < 136; i++ {
+		for i := 0; i < 64; i++ {
 			for j := 0; j < 8; j++ {
 				circuit.P[k][i*8+j] = int((data[i] >> j) & 1)
 			}
@@ -327,7 +333,14 @@ func main() {
 		panic("gg")
 	}
 
-	if !test.CheckCircuit(c, wit) {
+	out := test.EvalCircuit(c, wit)
+	fail := false
+	for i := 0; i <= NHashes*256; i++ {
+		if out[i].Uint64() == 1 {
+			fail = true
+		}
+	}
+	if fail {
 		panic("gg")
 	}
 
@@ -339,10 +352,14 @@ func main() {
 		panic("gg")
 	}
 
-	for i := 0; i < 10; i++ {
-		if !test.CheckCircuit(c, wit) {
-			return
+	out = test.EvalCircuit(c, wit)
+	done := false
+	for i := 0; i <= NHashes*256; i++ {
+		if out[i].Uint64() == 1 {
+			done = true
 		}
 	}
-	panic("should fail, but it succeed, please check the code, or you are just too lucky")
+	if !done {
+		panic("should fail")
+	}
 }

ecgo/layered/serialize.go

@@ -10,7 +10,7 @@ import (
 func (rc *RootCircuit) Serialize() []byte {
 	o := utils.OutputBuf{}
 	zero := big.NewInt(0)
-	o.AppendUint64(3842777012604389699)
+	o.AppendUint64(3770719418566461763)
 	o.AppendBigInt(rc.Field)
 	o.AppendUint64(uint64(len(rc.Circuits)))
 	for _, c := range rc.Circuits {
@@ -59,6 +59,7 @@ func (rc *RootCircuit) Serialize() []byte {
 				o.AppendBigInt(cst.Coef)
 			}
 		}
+		o.AppendUint64(0)
 		o.AppendUint64(uint64(len(randomCoefIdx)))
 		for _, idx := range randomCoefIdx {
 			o.AppendUint64(uint64(idx))
@@ -73,7 +74,7 @@ func (rc *RootCircuit) Serialize() []byte {
 
 func DeserializeRootCircuit(buf []byte) *RootCircuit {
 	in := utils.NewInputBuf(buf)
-	if in.ReadUint64() != 3842777012604389699 {
+	if in.ReadUint64() != 3770719418566461763 {
 		panic("invalid file header")
 	}
 	rc := &RootCircuit{}
@@ -118,6 +119,7 @@ func DeserializeRootCircuit(buf []byte) *RootCircuit {
 			c.Cst[j].Out = in.ReadUint64()
 			c.Cst[j].Coef = in.ReadBigInt()
 		}
+		in.ReadUint64()
 		nbRandomCoef := in.ReadUint64()
 		randomCoefIdx := make([]int, nbRandomCoef)
 		for j := uint64(0); j < nbRandomCoef; j++ {

expander_compiler/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2021"
 [profile.test]
 opt-level = 3
 
+[profile.dev]
+opt-level = 3
+
 [dependencies]
 rand.workspace = true
 uint.workspace = true

expander_compiler/src/builder/final_build_opt.rs

@@ -3,7 +3,7 @@ use std::collections::{BinaryHeap, HashMap};
 use crate::{
     circuit::{
         config::Config,
-        costs::{cost_of_compress, cost_of_multiply, cost_of_possible_references},
+        costs::{cost_of_compress, cost_of_multiply, cost_of_possible_references, cost_of_relay},
         ir::{
             common::Instruction,
             dest::{
@@ -117,7 +117,7 @@ impl<C: Config> Builder<C> {
     fn to_single(&mut self, expr: Expression<C>) -> Expression<C> {
         let (e, coef, constant) = strip_constants(&expr);
         if e.len() == 1 && e.degree() <= 1 {
-            return expr.clone();
+            return expr;
         }
         let idx = self.stripped_mid_vars.add(&e);
         if idx == self.mid_var_coefs.len() {
@@ -132,6 +132,19 @@ impl<C: Config> Builder<C> {
         return unstrip_constants_single(idx, coef, constant, &self.mid_var_coefs[idx]);
     }
 
+    fn try_to_single(&self, expr: Expression<C>) -> Expression<C> {
+        let (e, coef, constant) = strip_constants(&expr);
+        if e.len() == 1 && e.degree() <= 1 {
+            return expr;
+        }
+        match self.stripped_mid_vars.try_get_idx(&e) {
+            Some(idx) => {
+                return unstrip_constants_single(idx, coef, constant, &self.mid_var_coefs[idx]);
+            }
+            None => expr,
+        }
+    }
+
     fn to_really_single(&mut self, e: Expression<C>) -> usize {
         if e.len() == 1 && e.degree() == 1 && e[0].coef == C::CircuitField::one() {
             match e[0].vars {
@@ -153,13 +166,16 @@ impl<C: Config> Builder<C> {
         if coef == self.mid_var_coefs[idx].k && constant == self.mid_var_coefs[idx].b {
             return idx;
         }
+        let e = self.to_single(e);
         let idx = self.stripped_mid_vars.add(&e);
-        self.mid_var_coefs.push(MidVarCoef {
-            k: C::CircuitField::one(),
-            kinv: C::CircuitField::one(),
-            b: C::CircuitField::zero(),
-        });
-        self.mid_var_layer.push(self.layer_of_expr(&e) + 1);
+        if idx == self.mid_var_coefs.len() {
+            self.mid_var_coefs.push(MidVarCoef {
+                k: C::CircuitField::one(),
+                kinv: C::CircuitField::one(),
+                b: C::CircuitField::zero(),
+            });
+            self.mid_var_layer.push(self.layer_of_expr(&e) + 1);
+        }
         idx
     }
 
@@ -198,14 +214,15 @@ impl<C: Config> Builder<C> {
 
     fn lin_comb_inner<F: Fn(usize) -> C::CircuitField>(
         &mut self,
-        vars: Vec<Expression<C>>,
+        mut vars: Vec<Expression<C>>,
         var_coef: F,
     ) -> Expression<C> {
         if vars.len() == 0 {
             return Expression::default();
         }
+        let vars: Vec<Expression<C>> = vars.drain(..).map(|e| self.try_to_single(e)).collect();
         if vars.len() == 1 {
-            return vars[0].mul_constant(var_coef(0));
+            return self.layered_add(vars[0].mul_constant(var_coef(0)).to_terms());
         }
         let mut heap: BinaryHeap<LinMeta> = BinaryHeap::new();
         for l_id in 0..vars.len() {
@@ -255,7 +272,20 @@ impl<C: Config> Builder<C> {
 
     fn layered_add(&mut self, mut terms: Vec<Term<C>>) -> Expression<C> {
         if terms.len() <= 1 {
-            return Expression::from_terms(terms);
+            return Expression::from_terms_sorted(terms);
+        }
+        let min_layer = terms
+            .iter()
+            .map(|term| self.layer_of_varspec(&term.vars))
+            .min()
+            .unwrap();
+        let max_layer = terms
+            .iter()
+            .map(|term| self.layer_of_varspec(&term.vars))
+            .max()
+            .unwrap();
+        if min_layer == max_layer {
+            return Expression::from_terms_sorted(terms);
         }
         terms.sort_by(|a, b| {
             let la = self.layer_of_varspec(&a.vars);
@@ -281,8 +311,10 @@ impl<C: Config> Builder<C> {
 
     fn mul_vec(&mut self, vars: &Vec<usize>) -> Expression<C> {
         assert!(vars.len() >= 2);
-        let mut exprs: Vec<Expression<C>> =
-            vars.iter().map(|&v| self.in_var_exprs[v].clone()).collect();
+        let mut exprs: Vec<Expression<C>> = vars
+            .iter()
+            .map(|&v| self.try_to_single(self.in_var_exprs[v].clone()))
+            .collect();
         while exprs.len() > 1 {
             let mut exprs_pos: Vec<usize> = (0..exprs.len()).collect();
             exprs_pos.sort_by(|a, b| {
@@ -335,20 +367,38 @@ impl<C: Config> Builder<C> {
             let dcnt2 = expr2.count_of_degrees();
             assert!(dcnt1[2] == 0);
             assert!(dcnt2[2] == 0);
-            let cost_direct = cost_of_multiply::<C>(dcnt1[0], dcnt1[1], dcnt2[0], dcnt2[1]);
-            let cost_compress_v1 =
+            let v1layer = self.layer_of_expr(&expr1);
+            let v2layer = self.layer_of_expr(&expr2);
+            let mut cost_direct = cost_of_multiply::<C>(dcnt1[0], dcnt1[1], dcnt2[0], dcnt2[1]);
+            let mut cost_compress_v1 =
                 cost_of_multiply::<C>(0, 1, dcnt2[0], dcnt2[1]) + cost_of_compress::<C>(&dcnt1);
-            let cost_compress_v2 =
+            let mut cost_compress_v2 =
                 cost_of_multiply::<C>(dcnt1[0], dcnt1[1], 0, 1) + cost_of_compress::<C>(&dcnt2);
             let cost_compress_both = cost_of_multiply::<C>(0, 1, 0, 1)
                 + cost_of_compress::<C>(&dcnt1)
                 + cost_of_compress::<C>(&dcnt2);
-            if cost_compress_v1
-                .min(cost_compress_v2)
-                .min(cost_compress_both)
-                < cost_direct
-            {
-                if cost_compress_v1 < cost_compress_v2.max(cost_compress_both) {
+            let (compress_some, compress_1) = if v1layer == v2layer {
+                (
+                    cost_compress_v1
+                        .min(cost_compress_v2)
+                        .min(cost_compress_both)
+                        < cost_direct,
+                    cost_compress_v1 < cost_compress_v2.min(cost_compress_both),
+                )
+            } else {
+                cost_direct += cost_of_relay::<C>(v1layer, v2layer);
+                cost_compress_v1 += cost_of_relay::<C>(v1layer + 1, v2layer);
+                cost_compress_v2 += cost_of_relay::<C>(v1layer, v2layer + 1);
+                if cost_compress_v1 < cost_direct {
+                    (expr1.len() > 2, true)
+                } else if cost_compress_v2 < cost_direct {
+                    (expr2.len() > 2, false)
+                } else {
+                    (false, false)
+                }
+            };
+            if compress_some {
+                if compress_1 {
                     exprs.push(self.to_single(expr1));
                     exprs.push(expr2);
                 } else {
@@ -378,7 +428,9 @@ impl<C: Config> Builder<C> {
             + cost_of_possible_references::<C>(&[0, 1, 0], ref_count.add, ref_count.mul);
         should_compress |= cost_compress < cost_no_compress;
         should_compress &= e.degree() > 0;
-        if should_compress {
+        if should_compress && false {
+            // Currently, this don't consider the cost of relay, so it's disabled
+            // TODO: fix this
             let es = self.to_single(e);
             self.in_var_exprs.push(es);
         } else {
@@ -710,6 +762,7 @@ mod tests {
             config.seed = i + 100000;
             let root = super::InRootCircuit::<C>::random(&config);
             assert_eq!(root.validate(), Ok(()));
+            let (root, _) = root.remove_unreachable();
             match super::process(&root) {
                 Ok(root_processed) => {
                     assert_eq!(root_processed.validate(), Ok(()));

expander_compiler/src/circuit/config.rs

@@ -7,11 +7,13 @@ pub trait Config: Default + Clone + Ord + Debug + Hash + Copy + 'static {
 
     const CONFIG_ID: usize;
 
-    const COST_INPUT: usize = 100;
+    const COST_INPUT: usize = 1000;
     const COST_VARIABLE: usize = 100;
     const COST_MUL: usize = 10;
     const COST_ADD: usize = 3;
     const COST_CONST: usize = 3;
+
+    const ENABLE_RANDOM_COMBINATION: bool = true;
 }
 
 #[derive(Default, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -39,4 +41,6 @@ impl Config for GF2Config {
     type CircuitField = crate::field::gf2::GF2;
 
     const CONFIG_ID: usize = 3;
+
+    const ENABLE_RANDOM_COMBINATION: bool = false;
 }

expander_compiler/src/circuit/costs.rs

@@ -27,3 +27,7 @@ pub fn cost_of_possible_references<C: Config>(
         + C::COST_ADD * (deg_cnt[1] * ref_add + deg_cnt[0] * ref_mul)
         + C::COST_MUL * (deg_cnt[2] * ref_add + (deg_cnt[1] + deg_cnt[2] * 2) * ref_mul)
 }
+
+pub fn cost_of_relay<C: Config>(v1_layer: usize, v2_layer: usize) -> usize {
+    (v1_layer as isize - v2_layer as isize).abs() as usize * (C::COST_VARIABLE + C::COST_ADD)
+}