regularity analysis

benches/msm.rs

@@ -15,7 +15,10 @@ use pasta_curves::{
     group::ff::{Field, PrimeField},
     pallas,
 };
-use pasta_msm::{self, utils::{collect, compress, CommitmentKey}};
+use pasta_msm::{
+    self,
+    utils::{collect, compress, new_compress, CommitmentKey},
+};
 use rand::thread_rng;
 
 #[cfg(feature = "cuda")]
@@ -100,8 +103,13 @@ fn criterion_benchmark(c: &mut Criterion) {
         // }
 
         for i in 0..32 {
-let witness_i = read_abomonated::<Vec<<pallas::Scalar as PrimeField>::Repr>>(i.to_string()).unwrap();
-let mut witness_i = unsafe { std::mem::transmute::<Vec<_>, Vec<pallas::Scalar>>(witness_i) };
+            let witness_i = read_abomonated::<
+                Vec<<pallas::Scalar as PrimeField>::Repr>,
+            >(i.to_string())
+            .unwrap();
+            let mut witness_i = unsafe {
+                std::mem::transmute::<Vec<_>, Vec<pallas::Scalar>>(witness_i)
+            };
 
             let witness_n = witness_i.len();
 
@@ -157,18 +165,24 @@ let mut witness_i = unsafe { std::mem::transmute::<Vec<_>, Vec<pallas::Scalar>>(
                 },
             );
 
+            group.bench_function(
+                BenchmarkId::new("lurkrs new_compress", &name),
+                |b| {
+                    b.iter(|| {
+                        let _ = new_compress(&points[..witness_n], &witness_i);
+                    })
+                },
+            );
+
             group.bench_function(
                 BenchmarkId::new("lurkrs with compressed", &name),
                 |b| {
                     b.iter(|| {
                         let map = collect(&witness_i);
-                        let (com_i, com_points) = compress(&points[..witness_n], map);
+                        let (com_i, com_points) =
+                            compress(&points[..witness_n], map);
                         let com_n = com_i.len();
-                        let _ = pasta_msm::pallas(
-                            &com_points,
-                            &com_i,
-                            com_n,
-                        );
+                        let _ = pasta_msm::pallas(&com_points, &com_i, com_n);
                     })
                 },
             );

examples/abomonated_msm.rs

@@ -1,15 +1,28 @@
-use std::{io::{Read, BufWriter, Write}, time::Instant, collections::HashMap};
+#![allow(unused)]
+use std::{
+    collections::HashMap,
+    io::{BufWriter, Read, Write},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Mutex,
+    },
+    time::Instant,
+};
 
 use abomonation::Abomonation;
+use dashmap::DashMap;
 use num::{BigInt, Num, Zero};
 use pasta_curves::{
     group::ff::{Field, PrimeField},
     pallas,
 };
-use pasta_msm::utils::{gen_scalars, CommitmentKey};
+use pasta_msm::utils::collect;
 
 use plotters::prelude::*;
-use rayon::{iter::{IntoParallelRefIterator, ParallelIterator}, slice::ParallelSliceMut};    
+use rayon::{
+    iter::{IntoParallelIterator, ParallelIterator},
+    slice::ParallelSliceMut,
+};
 use std::fs::OpenOptions;
 use std::io::BufReader;
 
@@ -31,7 +44,7 @@ fn plot_scalars(
         .y_label_area_size(40)
         .margin(5)
         .caption(out_file, ("sans-serif", 50.0))
-        .build_cartesian_2d((0u32..1000u32).into_segmented(), 0u32..50000u32)?;
+        .build_cartesian_2d((0u32..1000u32).into_segmented(), 0u32..50u32)?;
 
     chart
         .configure_mesh()
@@ -63,7 +76,6 @@ fn plot_scalars(
 }
 
 fn read_abomonated<T: Abomonation + Clone>(name: String) -> std::io::Result<T> {
-
     let arecibo = home::home_dir().unwrap().join(".arecibo_witness");
 
     let data = OpenOptions::new()
@@ -80,86 +92,129 @@ fn read_abomonated<T: Abomonation + Clone>(name: String) -> std::io::Result<T> {
     Ok(data.clone())
 }
 
+fn get_witness(i: usize) -> Vec<pallas::Scalar> {
+    let witness_i =
+        read_abomonated::<Vec<<pallas::Scalar as PrimeField>::Repr>>(
+            i.to_string(),
+        )
+        .unwrap();
+    unsafe { std::mem::transmute::<Vec<_>, Vec<pallas::Scalar>>(witness_i) }
+}
+
 struct Stats {
     count: usize,
     total: usize,
 }
 
-fn write_stats(scalars: &[pallas::Scalar], out_file: &str) -> std::io::Result<()> {
+fn write_stats(
+    scalars: &[pallas::Scalar],
+    out_file: &str,
+) -> std::io::Result<()> {
     let stats = OpenOptions::new()
         .read(true)
         .write(true)
+        .truncate(true)
         .create(true)
         .open(out_file)?;
 
     let mut writer = BufWriter::new(stats);
-    let mut hist: HashMap<BigInt, usize> = HashMap::new();
+    let map = collect(&scalars);
 
-    scalars.iter().for_each(|x| {
-        let x = format!("{:?}", x);
-        let num = BigInt::from_str_radix(&x[2..], 16).unwrap();
-        *hist.entry(num).or_insert(0) += 1;
+    let mut entries: Vec<_> = map.into_par_iter().collect();
+    entries.par_sort_by(|a, b| {
+        let len_cmp = b.1.len().cmp(&a.1.len());
+        if len_cmp == std::cmp::Ordering::Equal {
+            a.1[0].cmp(&b.1[0])
+        } else {
+            len_cmp
+        }
     });
 
-    let mut entries: Vec<_> = hist.par_iter().collect();
-    entries.par_sort_by(|a, b| b.1.cmp(a.1));
-
-    let mut stats_20 = Stats { count: 0, total: 0};
-    let mut stats_100 = Stats { count: 0, total: 0};
-    let mut stats_500 = Stats { count: 0, total: 0};
-    let mut stats_1000 = Stats { count: 0, total: 0};
-    for (_, value) in entries.iter() {
-        if **value > 1000 {
+    let mut stats_20 = Stats { count: 0, total: 0 };
+    let mut stats_100 = Stats { count: 0, total: 0 };
+    let mut stats_500 = Stats { count: 0, total: 0 };
+    let mut stats_1000 = Stats { count: 0, total: 0 };
+    for (_, indices) in entries.iter() {
+        let len = indices.len();
+        if len > 1000 {
             stats_1000.count += 1;
-            stats_1000.total += *value;
-        } 
-        if **value > 500 {
+            stats_1000.total += len;
+        }
+        if len > 500 {
             stats_500.count += 1;
-            stats_500.total += *value;
-        } 
-        if **value > 100 {
+            stats_500.total += len;
+        }
+        if len > 100 {
             stats_100.count += 1;
-            stats_100.total += *value;
-        } 
-        if **value > 20 {
+            stats_100.total += len;
+        }
+        if len > 20 {
             stats_20.count += 1;
-            stats_20.total += *value;
-        } 
+            stats_20.total += len;
+        }
     }
 
     writeln!(&mut writer, "scalars length: {}", scalars.len())?;
     let p1000 = 100.0 * (stats_1000.total as f64 / scalars.len() as f64);
     let p500 = 100.0 * (stats_500.total as f64 / scalars.len() as f64);
     let p100 = 100.0 * (stats_100.total as f64 / scalars.len() as f64);
-    let p50 = 100.0 * (stats_20.total as f64 / scalars.len() as f64);
-    writeln!(&mut writer, "freq>1000 (count, total, %): {:>10}, {:>10}, {:>10.2}%", stats_1000.count, stats_1000.total, p1000)?;
-    writeln!(&mut writer, "freq>500  (count, total, %): {:>10}, {:>10}, {:>10.2}%", stats_500.count, stats_500.total, p500)?;
-    writeln!(&mut writer, "freq>100  (count, total, %): {:>10}, {:>10}, {:>10.2}%", stats_100.count, stats_100.total, p100)?;
-    writeln!(&mut writer, "freq>20   (count, total, %): {:>10}, {:>10}, {:>10.2}%", stats_20.count, stats_20.total, p50)?;
+    let p20 = 100.0 * (stats_20.total as f64 / scalars.len() as f64);
+    writeln!(
+        &mut writer,
+        "freq>1000 (count, total, %): {:>10}, {:>10}, {:>10.2}%",
+        stats_1000.count, stats_1000.total, p1000
+    )?;
+    writeln!(
+        &mut writer,
+        "freq>500  (count, total, %): {:>10}, {:>10}, {:>10.2}%",
+        stats_500.count, stats_500.total, p500
+    )?;
+    writeln!(
+        &mut writer,
+        "freq>100  (count, total, %): {:>10}, {:>10}, {:>10.2}%",
+        stats_100.count, stats_100.total, p100
+    )?;
+    writeln!(
+        &mut writer,
+        "freq>20   (count, total, %): {:>10}, {:>10}, {:>10.2}%",
+        stats_20.count, stats_20.total, p20
+    )?;
     writeln!(&mut writer, "")?;
-    writeln!(&mut writer, "top 100 values:")?;
+    writeln!(&mut writer, "top values:")?;
 
     let mut count = 0;
+    let mut prev: Vec<usize> = vec![];
     // Print sorted (key, value) pairs
-    for (key, value) in entries {
-        if count < 100 {
-            writeln!(&mut writer, "{:>2}, {:>64}, {:?}", count, key.to_str_radix(16), value)?;
+    for (key, mut indices) in entries {
+        let len = indices.len();
+        if len > 1 {
+            writeln!(&mut writer, "{:>4}, {:>64?}, {:?}", count, key, len)?;
+            writeln!(
+                &mut writer,
+                "      {:?}",
+                &indices[..std::cmp::min(len, 20)]
+            )?;
+            if indices.len() == prev.len() {
+                let offsets = indices
+                    .iter()
+                    .zip(prev.iter())
+                    .map(|(a, b)| a - b)
+                    .collect::<Vec<_>>();
+                let equal = offsets.iter().all(|x| x == &offsets[0]);
+                if equal {
+                    writeln!(&mut writer, "      offset: {}", offsets[0] as isize);
+                }
+            }
         }
+        prev = indices;
         count += 1;
     }
     writer.flush()?;
     Ok(())
 }
 
-/// cargo run --release --example abomonated
-fn main() {
-    #[cfg(feature = "cuda")]
-    if unsafe { cuda_available() } {
-        unsafe { pasta_msm::CUDA_OFF = false };
-    }
-
+fn generate_stats_and_plots() {
     for i in 0..32 {
-
         let witness_i = read_abomonated::<
             Vec<<pallas::Scalar as PrimeField>::Repr>,
         >(i.to_string())
@@ -168,17 +223,103 @@ fn main() {
             std::mem::transmute::<Vec<_>, Vec<pallas::Scalar>>(witness_i)
         };
 
-        let witness_n = witness_i.len();
+        // let witness_n = witness_i.len();
 
-        if witness_n < 1_000_000 {
-            continue;
-        }
-        
-        // let out_file = format!("plots/lurkrs_{i}.png");
-        // plot_scalars(&witness_i, &out_file).unwrap();
-        let out_stats = format!("stats/{i}.txt");
+        // if witness_n < 1_000_000 {
+        //     continue;
+        // }
+
+        let out_file = format!("plots/rc=1/{i}.png");
+        plot_scalars(&witness_i, &out_file).unwrap();
+        let out_stats = format!("stats/rc=1/{i}.txt");
         write_stats(&witness_i, &out_stats).unwrap();
 
         println!("done {i}");
     }
 }
+
+fn generate_subset_analysis() {
+    for i in (3..32).step_by(4) {
+        let out_file = format!("subset_stats/rc=1/{}", i);
+        let subsets = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .truncate(true)
+            .create(true)
+            .open(out_file)
+            .unwrap();
+        let mut writer = Mutex::new(BufWriter::new(subsets));
+
+        let curr = i;
+        let next = i + 4;
+
+        let witness_curr = get_witness(curr);
+        let witness_next = get_witness(next);
+
+        let map_curr = collect(&witness_curr);
+        let map_next = collect(&witness_next);
+
+        let mut total = AtomicUsize::new(0);
+
+        map_curr.into_par_iter().for_each(|(x, indices_curr)| {
+            for data in map_next.iter() {
+                let y = data.key();
+                let indices_next = data.value();
+                let n = indices_curr.len();
+                if indices_curr[0] == indices_next[0] && n > 1 {
+                    let mut inter = vec![];
+                    let mut curr = vec![];
+                    let mut next = vec![];
+                    for (a, b) in indices_curr.iter().zip(indices_next) {
+                        if a == b {
+                            inter.push(*a);
+                        } else {
+                            curr.push(*a);
+                            next.push(*b);
+                        }
+                    }
+                    let mut writer = writer.lock().unwrap();
+                    writeln!(&mut writer, ">>> fuzzy head equality:").unwrap();
+                    writeln!(&mut writer, "    x:       {:?}", x).unwrap();
+                    writeln!(&mut writer, "    y:       {:?}", y).unwrap();
+                    writeln!(
+                        &mut writer,
+                        "    inter: {:?}",
+                        &inter[..std::cmp::min(20, inter.len())]
+                    )
+                    .unwrap();
+                    writeln!(
+                        &mut writer,
+                        "    - curr: {:?}",
+                        &curr[..std::cmp::min(20, curr.len())]
+                    )
+                    .unwrap();
+                    writeln!(
+                        &mut writer,
+                        "    + next: {:?}",
+                        &next[..std::cmp::min(20, next.len())]
+                    )
+                    .unwrap();
+                    writeln!(&mut writer, "    length:  {:?}", n).unwrap();
+                    total.fetch_add(n, Ordering::Relaxed);
+                    writeln!(&mut writer,);
+                }
+            }
+        });
+
+        let mut writer = writer.get_mut().unwrap();
+        writeln!(&mut writer, "total: {:?}", total).unwrap();
+        println!("done {i}");
+    }
+}
+
+/// cargo run --release --example abomonated
+fn main() {
+    #[cfg(feature = "cuda")]
+    if unsafe { cuda_available() } {
+        unsafe { pasta_msm::CUDA_OFF = false };
+    }
+
+    generate_stats_and_plots()
+    // generate_subset_analysis()
+}