Implement the proof of concept of a chunked approach (#268)

* Implement the proof of concept of a chunked approach

* fix: test data 'entry_64.csv' with random balances

* Add BALANCES_INDEX constant and update comments

---------

Co-authored-by: sifnoc <[email protected]>

csv/entry_64.csv

@@ -1,65 +1,65 @@
 username,balance_ETH_ETH,balance_USDT_ETH
-dxGaEAii,11888,41163
-MBlfbBGI,67823,18651
-lAhWlEWZ,18651,2087
-nuZweYtO,22073,55683
-gbdSwiuY,34897,83296
-RZNneNuP,83296,16881
-YsscHXkp,31699,35479
-RkLzkDun,2087,79731
-HlQlnEYI,30605,11888
-RqkZOFYe,16881,14874
-NjCSRAfD,41163,67823
-pHniJMQY,14874,22073
-dOGIMzKR,10032,10032
-HfMDmNLp,55683,34897
-xPLKzCBl,79731,30605
-AtwIxZHo,35479,31699
-aaGaEAaa,11888,41163
-bblfbBGI,67823,18651
-cchWlEWZ,18651,2087
-ddZweYtO,22073,55683
-eedSwiuY,34897,83296
-ffNneNuP,83296,16881
-ggscHXkp,31699,35479
-hhLzkDun,2087,79731
-iiQlnEYI,30605,11888
-llkZOFYe,16881,14874
-mmCSRAfD,41163,67823
-nnniJMQY,14874,22073
-ooGIMzKR,10032,10032
-ppMDmNLp,55683,34897
-qqLKzCBl,79731,30605
-rrwIxZHo,35479,31699
-a1GaEAaa,11888,41163
-b2lfbBGI,67823,18651
-c3hWlEWZ,18651,2087
-d4ZweYtO,22073,55683
-e5dSwiuY,34897,83296
-f6NneNuP,83296,16881
-g7scHXkp,31699,35479
-h8LzkDun,2087,79731
-i9QlnEYI,30605,11888
-l0kZOFYe,16881,14874
-m1CSRAfD,41163,67823
-n2niJMQY,14874,22073
-o3GIMzKR,10032,10032
-p4MDmNLp,55683,34897
-q5LKzCBl,79731,30605
-r6wIxZHo,35479,31699
-a17aEAaa,11888,41163
-b28fbBGI,67823,18651
-c39WlEWZ,18651,2087
-d40weYtO,22073,55683
-e51SwiuY,34897,83296
-f62neNuP,83296,16881
-g73cHXkp,31699,35479
-h84zkDun,2087,79731
-i95lnEYI,30605,11888
-l06ZOFYe,16881,14874
-m17SRAfD,41163,67823
-n28iJMQY,14874,22073
-o39IMzKR,10032,10032
-p40DmNLp,55683,34897
-q51KzCBl,79731,30605
-r62IxZHo,35479,31699
+lQqmRJdg,12388,32294
+aWDvCEiS,9693,16236
+xnsyHKax,38938,67192
+TeWkyMTK,65809,32114
+KoAjYZbg,43955,15060
+BvPoRkOl,56729,26812
+gZKobVvp,44121,14034
+TWrbSQEs,76672,64256
+ETShTNyr,8101,83099
+ZhczOKsa,41787,15680
+hXWusLiI,16063,48839
+NPJvNZnI,58437,29615
+hHPFzFhT,72496,7798
+HhQNIETc,30750,14476
+yEuILJNE,17503,12454
+Squeylsr,8765,2983
+OvLSqsvt,64390,62263
+EvHMzKbu,84747,66310
+NcIPonBQ,67156,87921
+AqVWwRbT,18974,38473
+hJBGnXrB,7057,85818
+QfGnaeip,19512,30398
+BJZjoRyQ,33258,64197
+nMkYmaKL,74350,62280
+WUgvEEqq,77422,17256
+YlKwJzXP,82114,37724
+FWWYnWJe,27057,74057
+InXPwhCF,35937,88954
+dddmaToF,3261,20421
+amuobdlW,44195,5439
+ttsnlNWp,58459,50750
+zOKIdByy,32597,19330
+gcTKDudW,29652,77180
+hOqGkBYW,37493,28546
+DhboScmx,64879,77838
+jOwrDAqq,85178,18293
+aAdhWvot,27156,57687
+UyBdPJqB,42003,25321
+tuvfHyXy,32258,89286
+qcHjtmct,30416,1487
+IiCgtzNf,34391,10698
+grSDJvEC,24633,19852
+qdFUCFAa,6586,20407
+RTcBVkAp,89200,42858
+yJqccDKr,61092,24056
+zqxOHXcJ,71826,53820
+xkjzJYUE,24503,71001
+slrXGVgT,67494,28243
+ZHCfpHry,54977,68392
+PsWWISCN,50035,51087
+vYvNfjxM,72096,62588
+zXvkQZxE,10632,82593
+retKdNHy,72475,27380
+pYSKrVjZ,69450,64384
+mEIfsRZu,59127,81500
+QuUzDtzJ,58949,84898
+eCVMnmfE,77210,31031
+nPoFfqUY,86755,32206
+TDzGkSlY,30306,75914
+KODqnvSt,71746,60682
+dYseTKwI,45205,51285
+dMypqzKK,9543,21413
+LLzMkgKr,58363,31840
+smRTQZYt,88641,23869

kzg_prover/README.md

@@ -94,3 +94,23 @@ To run the quick benchmarks with the range check disabled (K=9..12), use the fol
 ```shell
 cargo bench --features "no_range_check"
 ```
+
+## Chunked Univariate Grand Sum Example
+
+the following technique is proposed to further improve the performance of the univariate grand sum version of Summa:
+
+1. Split the user base into chunks;
+2. Generate the zkSNARK range proof for all the users of each chunk (one proof per chunk) alongside with the advice polynomials;
+3. Generate the proofs of inclusion of a user into a specific chunk;
+4. Prove the grand total across the chunks by performing the following:
+   1. Add together the chunk polynomials generated in step 2;
+   2. Add their corresponding KZG commitments together using the homomorphic property of the KZG commitment;
+   3. Generate the opening proof of the grand sum for the resulting polynomial from step 4.1 against the commitment from step 4.3
+
+Step 4 of the algorithm establishes the relation between the chunks containing individual user liabilities and the grand sum of all user liabilities. The proof of inclusion generation in step 3 should be carried out using the amortized KZG approach in the similar fashion as in the non-chunked version of Summa.
+
+The proof of concept implementation of the suggested approach can be found in the [example file](kzg_prover/examples/chunked_univariate_grand_sum.rs). To execute the example, use the command:
+
+```shell
+cargo run --release --example chunked_univariate_grand_sum
+```

kzg_prover/examples/chunked_univariate_grand_sum.rs

@@ -0,0 +1,179 @@
+#![feature(generic_const_exprs)]
+use std::error::Error;
+
+use halo2_proofs::halo2curves::bn256::{Fr as Fp, G1Affine};
+use halo2_proofs::halo2curves::group::cofactor::CofactorCurveAffine;
+use halo2_proofs::halo2curves::group::Curve;
+use halo2_proofs::transcript::TranscriptRead;
+use halo2_proofs::{
+    arithmetic::Field, halo2curves::bn256::Bn256, poly::kzg::commitment::KZGCommitmentScheme,
+};
+use halo2_solidity_verifier::Keccak256Transcript;
+use num_bigint::BigUint;
+
+use summa_solvency::circuits::utils::generate_setup_artifacts;
+use summa_solvency::{
+    circuits::{
+        univariate_grand_sum::{NoRangeCheckConfig, UnivariateGrandSum},
+        utils::full_prover,
+    },
+    cryptocurrency::Cryptocurrency,
+    entry::Entry,
+    utils::{
+        amortized_kzg::{commit_kzg, create_naive_kzg_proof, verify_kzg_proof},
+        big_uint_to_fp, parse_csv_to_entries,
+    },
+};
+
+const K: u32 = 9;
+const N_CURRENCIES: usize = 2;
+const N_USERS_TOTAL: usize = 64;
+const N_USERS_CHUNK: usize = N_USERS_TOTAL / 2;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let path = "../csv/entry_64.csv";
+
+    let mut entries: Vec<Entry<N_CURRENCIES>> = vec![Entry::init_empty(); N_USERS_TOTAL];
+    let mut cryptos = vec![Cryptocurrency::init_empty(); N_CURRENCIES];
+
+    parse_csv_to_entries::<&str, N_CURRENCIES>(path, &mut entries, &mut cryptos).unwrap();
+
+    // Calculate total for all balance entries
+    let mut csv_total: Vec<BigUint> = vec![BigUint::from(0u32); N_CURRENCIES];
+
+    for entry in &entries {
+        for (i, balance) in entry.balances().iter().enumerate() {
+            csv_total[i] += balance;
+        }
+    }
+
+    // Split the user base into two equal chunks of N_USERS_TOTAL/2 each
+    let entries_first_chunk = entries[0..N_USERS_CHUNK].to_vec();
+    // Calculate the total for the first chunk
+    let mut csv_total_1: Vec<BigUint> = vec![BigUint::from(0u32); N_CURRENCIES];
+    for entry in &entries_first_chunk {
+        for (i, balance) in entry.balances().iter().enumerate() {
+            csv_total_1[i] += balance;
+        }
+    }
+    let entries_second_chunk = entries[N_USERS_CHUNK..].to_vec();
+    // Calculate the total for the second chunk
+    let mut csv_total_2: Vec<BigUint> = vec![BigUint::from(0u32); N_CURRENCIES];
+    for entry in &entries_second_chunk {
+        for (i, balance) in entry.balances().iter().enumerate() {
+            csv_total_2[i] += balance;
+        }
+    }
+    // Index of the advice polynomial to be used for the subsequent examples
+    const BALANCES_INDEX: usize = 1;
+    assert!(
+        &csv_total_1[BALANCES_INDEX - 1] + &csv_total_2[BALANCES_INDEX - 1]
+            == csv_total[BALANCES_INDEX - 1],
+        "The sum of the chunks' total should be equal to the grand total"
+    );
+
+    type CONFIG = NoRangeCheckConfig<N_CURRENCIES, N_USERS_CHUNK>;
+
+    let circuit_1 = UnivariateGrandSum::<N_USERS_CHUNK, N_CURRENCIES, CONFIG>::init_empty();
+    // Generate the setup artifacts using an empty circuit
+    let (params, pk, vk) = generate_setup_artifacts(K, None, &circuit_1).unwrap();
+
+    // Instantiate the actual circuits for the first and second chunk
+    let circuit_1 =
+        UnivariateGrandSum::<N_USERS_CHUNK, N_CURRENCIES, CONFIG>::init(entries_first_chunk);
+    let circuit_2 =
+        UnivariateGrandSum::<N_USERS_CHUNK, N_CURRENCIES, CONFIG>::init(entries_second_chunk);
+
+    // The zkSNARK proofs encode the balances of the first chunk and the second chunk
+    // in the corresponding advice polynomials
+    let (proof_1, advice_polys_1, _) = full_prover(&params, &pk, circuit_1.clone(), &[vec![]]);
+    let (proof_2, advice_polys_2, _) = full_prover(&params, &pk, circuit_2.clone(), &[vec![]]);
+
+    // Get the BALANCES_INDEX advice polynomial from each chunk
+    let f_poly_1 = advice_polys_1.advice_polys.get(BALANCES_INDEX).unwrap();
+    let f_poly_2 = advice_polys_2.advice_polys.get(BALANCES_INDEX).unwrap();
+
+    // These advice polynomials can then be used to independently produce the user inclusion KZG proofs.
+    // This allows to significantly speed up the inclusion proof by using smaller `N_USERS_CHUNK` size
+    // and parallelizing the proof generation.
+
+    // Take the KZG commitment of each chunk from the zkSNARK proof transcript
+    let mut transcript_1 = Keccak256Transcript::new(proof_1.as_slice());
+    let mut advice_commitments_1 = Vec::new();
+    (0..N_CURRENCIES + 1).for_each(|_| {
+        let point: G1Affine = transcript_1.read_point().unwrap();
+        advice_commitments_1.push(point);
+    });
+    let kzg_commitment_1 = advice_commitments_1[BALANCES_INDEX];
+    let mut transcript_2 = Keccak256Transcript::new(proof_2.as_slice());
+    let mut advice_commitments_2 = Vec::new();
+    (0..N_CURRENCIES + 1).for_each(|_| {
+        let point: G1Affine = transcript_2.read_point().unwrap();
+        advice_commitments_2.push(point);
+    });
+    let kzg_commitment_2 = advice_commitments_2[BALANCES_INDEX];
+    assert!(
+        kzg_commitment_1 != kzg_commitment_2,
+        "Commitments should be different"
+    );
+
+    // The homomorphic property of KZG commitments allows us to sum the individual chunk commitments
+    // to produce the KZG opening proof for the grand total
+    let kzg_commitment_sum = kzg_commitment_1 + kzg_commitment_2;
+
+    // First, add the polynomials together coefficient-wise
+    let domain = vk.get_domain();
+    let mut f_poly_total = domain.empty_coeff();
+
+    for (poly, value) in f_poly_total
+        .iter_mut()
+        .zip(f_poly_1.iter().zip(f_poly_2.iter()))
+    {
+        *poly = *value.0 + *value.1;
+    }
+
+    // Demonstrating the homomorphic property of KZG commitments. The sum of the KZG commitments
+    // to the chunk polynomials should be the same as the KZG commitment to the total polynomial
+    // that is a sum of the chunk polynomials
+    let kzg_commitment_total = commit_kzg(&params, &f_poly_total);
+    assert!(
+        kzg_commitment_sum.to_affine() == kzg_commitment_total.to_affine(),
+        "Commitments should be equal"
+    );
+
+    let poly_length = 1 << u64::from(K);
+
+    // We're opening the resulting polynomial at x = 0 and expect the constant coefficient
+    // to be equal to the grand total divided by the size of the polynomial
+    // thanks to the univariate grand sum property.
+    let challenge = Fp::ZERO;
+    // The expected evaluation of the polynomial at x = 0 is the grand total divided by the size of the polynomial
+    let eval =
+        big_uint_to_fp(&(csv_total[BALANCES_INDEX - 1])) * Fp::from(poly_length).invert().unwrap();
+    let kzg_proof = create_naive_kzg_proof::<KZGCommitmentScheme<Bn256>>(
+        &params,
+        &domain,
+        &f_poly_total,
+        challenge,
+        eval,
+    );
+
+    // KZG proof verification demonstrates that we can successfully verify the grand total
+    // after building the total KZG commitment from the chunk commitments
+    assert!(
+        verify_kzg_proof(&params, kzg_commitment_sum, kzg_proof, &challenge, &eval),
+        "KZG proof verification failed"
+    );
+    assert!(
+        !verify_kzg_proof(
+            &params,
+            kzg_commitment_sum,
+            kzg_proof,
+            &challenge,
+            &big_uint_to_fp(&BigUint::from(123u32)),
+        ),
+        "Invalid proof verification should fail"
+    );
+
+    Ok(())
+}