diff --git a/ceno_zkvm/src/expression.rs b/ceno_zkvm/src/expression.rs index a39f8a6a1..c2b523014 100644 --- a/ceno_zkvm/src/expression.rs +++ b/ceno_zkvm/src/expression.rs @@ -4,7 +4,6 @@ use std::{ cmp::max, fmt::Display, iter::{Product, Sum}, - mem::MaybeUninit, ops::{Add, AddAssign, Deref, Mul, MulAssign, Neg, Shl, ShlAssign, Sub, SubAssign}, }; @@ -756,12 +755,8 @@ impl WitIn { ) } - pub fn assign( - &self, - instance: &mut [MaybeUninit], - value: E::BaseField, - ) { - instance[self.id as usize] = MaybeUninit::new(value); + pub fn assign(&self, instance: &mut [E::BaseField], value: E::BaseField) { + instance[self.id as usize] = value; } } diff --git a/ceno_zkvm/src/gadgets/div.rs b/ceno_zkvm/src/gadgets/div.rs index 91f4b42f1..d3808349b 100644 --- a/ceno_zkvm/src/gadgets/div.rs +++ b/ceno_zkvm/src/gadgets/div.rs @@ -1,4 +1,4 @@ -use std::{fmt::Display, mem::MaybeUninit}; +use std::fmt::Display; use ff_ext::ExtensionField; @@ -53,7 +53,7 @@ impl DivConfig { pub fn assign_instance<'a>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, divisor: &Value<'a, u32>, quotient: &Value<'a, u32>, diff --git a/ceno_zkvm/src/gadgets/is_lt.rs b/ceno_zkvm/src/gadgets/is_lt.rs index ff0fe3989..c9dedaa03 100644 --- a/ceno_zkvm/src/gadgets/is_lt.rs +++ b/ceno_zkvm/src/gadgets/is_lt.rs @@ -1,4 +1,4 @@ -use std::{fmt::Display, mem::MaybeUninit}; +use std::fmt::Display; use ceno_emul::{SWord, Word}; use ff_ext::ExtensionField; @@ -52,7 +52,7 @@ impl AssertLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -106,7 +106,7 @@ impl IsLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -118,7 +118,7 @@ impl IsLtConfig { pub fn assign_instance_signed( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -184,7 +184,7 @@ impl InnerLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -202,7 +202,7 @@ impl InnerLtConfig { // TODO: refactor with the above function pub fn assign_instance_signed( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -256,7 +256,7 @@ impl AssertSignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -299,7 +299,7 @@ impl SignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -351,7 +351,7 @@ impl InnerSignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, diff --git a/ceno_zkvm/src/gadgets/is_zero.rs b/ceno_zkvm/src/gadgets/is_zero.rs index f7d749354..024ebf1bd 100644 --- a/ceno_zkvm/src/gadgets/is_zero.rs +++ b/ceno_zkvm/src/gadgets/is_zero.rs @@ -1,5 +1,3 @@ -use std::mem::MaybeUninit; - use ff_ext::ExtensionField; use goldilocks::SmallField; @@ -64,7 +62,7 @@ impl IsZeroConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], x: F, ) -> Result<(), ZKVMError> { let (is_zero, inverse) = if x.is_zero_vartime() { @@ -117,7 +115,7 @@ impl IsEqualConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], a: F, b: F, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/gadgets/signed_ext.rs b/ceno_zkvm/src/gadgets/signed_ext.rs index d1dc8ed62..3b6757bc3 100644 --- a/ceno_zkvm/src/gadgets/signed_ext.rs +++ b/ceno_zkvm/src/gadgets/signed_ext.rs @@ -7,7 +7,7 @@ use crate::{ witness::LkMultiplicity, }; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; #[derive(Debug)] pub struct SignedExtendConfig { @@ -84,7 +84,7 @@ impl SignedExtendConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, val: u64, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions.rs b/ceno_zkvm/src/instructions.rs index eb24b7008..b72401aec 100644 --- a/ceno_zkvm/src/instructions.rs +++ b/ceno_zkvm/src/instructions.rs @@ -1,5 +1,3 @@ -use std::mem::MaybeUninit; - use ceno_emul::StepRecord; use ff_ext::ExtensionField; use multilinear_extensions::util::max_usable_threads; @@ -7,20 +5,27 @@ use rayon::{ iter::{IndexedParallelIterator, ParallelIterator}, slice::ParallelSlice, }; +use std::sync::Arc; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, - scheme::constants::MIN_PAR_SIZE, witness::{LkMultiplicity, RowMajorMatrix}, }; -use ff::Field; pub mod riscv; +#[derive(Clone)] pub enum InstancePaddingStrategy { - Zero, + // Pads with default values of underlying type + // Usually zero, but check carefully + Default, + // Pads by repeating last row RepeatLast, + // Custom strategy consists of a closure + // `pad(i, j) = padding value for cell at row i, column j` + // pad should be able to cross thread boundaries + Custom(Arc u64 + Send + Sync>), } pub trait Instruction { @@ -38,7 +43,7 @@ pub trait Instruction { // assign single instance giving step from trace fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError>; @@ -56,7 +61,8 @@ pub trait Instruction { } .max(1); let lk_multiplicity = LkMultiplicity::default(); - let mut raw_witin = RowMajorMatrix::::new(steps.len(), num_witin); + let mut raw_witin = + RowMajorMatrix::::new(steps.len(), num_witin, Self::padding_strategy()); let raw_witin_iter = raw_witin.par_batch_iter_mut(num_instance_per_batch); raw_witin_iter @@ -73,35 +79,6 @@ pub trait Instruction { }) .collect::>()?; - let num_padding_instances = raw_witin.num_padding_instances(); - if num_padding_instances > 0 { - // Fill the padding based on strategy - - let padding_instance = match Self::padding_strategy() { - InstancePaddingStrategy::Zero => { - vec![MaybeUninit::new(E::BaseField::ZERO); num_witin] - } - InstancePaddingStrategy::RepeatLast if steps.is_empty() => { - tracing::debug!("No {} steps to repeat, using zero padding", Self::name()); - vec![MaybeUninit::new(E::BaseField::ZERO); num_witin] - } - InstancePaddingStrategy::RepeatLast => raw_witin[steps.len() - 1].to_vec(), - }; - - let num_padding_instance_per_batch = if num_padding_instances > 256 { - num_padding_instances.div_ceil(nthreads) - } else { - num_padding_instances - }; - raw_witin - .par_batch_iter_padding_mut(num_padding_instance_per_batch) - .with_min_len(MIN_PAR_SIZE) - .for_each(|row| { - row.chunks_mut(num_witin) - .for_each(|instance| instance.copy_from_slice(padding_instance.as_slice())); - }); - } - Ok((raw_witin, lk_multiplicity)) } } diff --git a/ceno_zkvm/src/instructions/riscv/arith.rs b/ceno_zkvm/src/instructions/riscv/arith.rs index 1e8c1be67..f6205130a 100644 --- a/ceno_zkvm/src/instructions/riscv/arith.rs +++ b/ceno_zkvm/src/instructions/riscv/arith.rs @@ -8,7 +8,6 @@ use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, instructions::Instruction, uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles R-Instructions that represent registers values as 2 * u16. #[derive(Debug)] @@ -88,7 +87,7 @@ impl Instruction for ArithInstruction], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/arith_imm.rs b/ceno_zkvm/src/instructions/riscv/arith_imm.rs index 2508000f2..c4c09629b 100644 --- a/ceno_zkvm/src/instructions/riscv/arith_imm.rs +++ b/ceno_zkvm/src/instructions/riscv/arith_imm.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::StepRecord; use ff_ext::ExtensionField; @@ -57,7 +57,7 @@ impl Instruction for AddiInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/b_insn.rs b/ceno_zkvm/src/instructions/riscv/b_insn.rs index c638d314f..646e9bd05 100644 --- a/ceno_zkvm/src/instructions/riscv/b_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/b_insn.rs @@ -13,7 +13,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; // Opcode: 1100011 // Funct3: @@ -88,7 +87,7 @@ impl BInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs b/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs index 4826c94bf..b8c96fbd1 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; @@ -72,7 +72,7 @@ impl Instruction for BeqCircuit { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/blt.rs b/ceno_zkvm/src/instructions/riscv/branch/blt.rs index c5e0798f2..dfe5995e1 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/blt.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/blt.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff_ext::ExtensionField; @@ -66,7 +66,7 @@ impl Instruction for BltCircuit { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/bltu.rs b/ceno_zkvm/src/instructions/riscv/branch/bltu.rs index 896bf19da..713158fdc 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/bltu.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/bltu.rs @@ -75,7 +75,7 @@ impl Instruction for BltuCircuit fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [std::mem::MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/config.rs b/ceno_zkvm/src/instructions/riscv/config.rs index cd13e1148..ef49b44f2 100644 --- a/ceno_zkvm/src/instructions/riscv/config.rs +++ b/ceno_zkvm/src/instructions/riscv/config.rs @@ -1,5 +1,3 @@ -use std::mem::MaybeUninit; - use crate::{expression::WitIn, set_val, utils::i64_to_base, witness::LkMultiplicity}; use goldilocks::SmallField; use itertools::Itertools; @@ -25,7 +23,7 @@ pub struct MsbInput<'a> { impl MsbInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &MsbConfig, lk_multiplicity: &mut LkMultiplicity, ) -> (u8, u8) { @@ -61,7 +59,7 @@ pub struct UIntLtuInput<'a> { impl UIntLtuInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &UIntLtuConfig, lk_multiplicity: &mut LkMultiplicity, ) -> bool { @@ -138,7 +136,7 @@ pub struct UIntLtInput<'a> { impl UIntLtInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &UIntLtConfig, lk_multiplicity: &mut LkMultiplicity, ) -> bool { diff --git a/ceno_zkvm/src/instructions/riscv/div.rs b/ceno_zkvm/src/instructions/riscv/div.rs index e9cb2e4ca..f162478fa 100644 --- a/ceno_zkvm/src/instructions/riscv/div.rs +++ b/ceno_zkvm/src/instructions/riscv/div.rs @@ -16,7 +16,6 @@ use crate::{ uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; use std::marker::PhantomData; pub struct ArithConfig { @@ -124,7 +123,7 @@ impl Instruction for ArithInstruction], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -173,7 +172,6 @@ mod test { use ceno_emul::{Change, InsnKind, StepRecord, Word, encode_rv32}; use goldilocks::GoldilocksExt2; use itertools::Itertools; - use multilinear_extensions::mle::IntoMLEs; use rand::Rng; use crate::{ @@ -238,7 +236,6 @@ mod test { MockProver::assert_with_expected_errors( &cb, &raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) diff --git a/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs b/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs index bb85ad7c1..1d98d1271 100644 --- a/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs @@ -19,7 +19,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// DummyInstruction can handle any instruction and produce its side-effects. pub struct DummyInstruction(PhantomData<(E, I)>); @@ -69,7 +68,7 @@ impl Instruction for DummyInstruction], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -214,7 +213,7 @@ impl DummyConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/ecall/halt.rs b/ceno_zkvm/src/instructions/riscv/ecall/halt.rs index 70338e3d6..d3d0d90a7 100644 --- a/ceno_zkvm/src/instructions/riscv/ecall/halt.rs +++ b/ceno_zkvm/src/instructions/riscv/ecall/halt.rs @@ -16,7 +16,7 @@ use crate::{ }; use ceno_emul::{StepRecord, Tracer}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct HaltConfig { ecall_cfg: EcallInstructionConfig, @@ -65,7 +65,7 @@ impl Instruction for HaltInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/ecall_insn.rs b/ceno_zkvm/src/instructions/riscv/ecall_insn.rs index 0457979c7..078223617 100644 --- a/ceno_zkvm/src/instructions/riscv/ecall_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/ecall_insn.rs @@ -12,7 +12,6 @@ use crate::{ }; use ceno_emul::{InsnKind::EANY, PC_STEP_SIZE, Platform, StepRecord, Tracer}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; pub struct EcallInstructionConfig { pub pc: WitIn, @@ -68,7 +67,7 @@ impl EcallInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/i_insn.rs b/ceno_zkvm/src/instructions/riscv/i_insn.rs index 65beb8c5f..52147cdec 100644 --- a/ceno_zkvm/src/instructions/riscv/i_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/i_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of I-type instructions: /// - PC, cycle, fetch. @@ -57,7 +56,7 @@ impl IInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/im_insn.rs b/ceno_zkvm/src/instructions/riscv/im_insn.rs index 6727f6628..36ad1a8a1 100644 --- a/ceno_zkvm/src/instructions/riscv/im_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/im_insn.rs @@ -9,7 +9,6 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; /// This config handle the common part of I-type Instruction (memory variant) /// - PC, cycle, fetch @@ -62,7 +61,7 @@ impl IMInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/insn_base.rs b/ceno_zkvm/src/instructions/riscv/insn_base.rs index 3f71b6b20..468eb3623 100644 --- a/ceno_zkvm/src/instructions/riscv/insn_base.rs +++ b/ceno_zkvm/src/instructions/riscv/insn_base.rs @@ -18,7 +18,6 @@ use crate::{ witness::LkMultiplicity, }; use ceno_emul::Tracer; -use core::mem::MaybeUninit; use std::{iter, marker::PhantomData}; #[derive(Debug)] @@ -59,7 +58,7 @@ impl StateInOut { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], // lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -107,7 +106,7 @@ impl ReadRS1 { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -161,7 +160,7 @@ impl ReadRS2 { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -217,7 +216,7 @@ impl WriteRD { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -275,7 +274,7 @@ impl ReadMEM { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -328,7 +327,7 @@ impl WriteMEM { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -437,7 +436,7 @@ impl MemAddr { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, addr: Word, ) -> Result<(), ZKVMError> { @@ -471,12 +470,12 @@ impl MemAddr { mod test { use goldilocks::{Goldilocks as F, GoldilocksExt2 as E}; use itertools::Itertools; - use multilinear_extensions::mle::IntoMLEs; use crate::{ ROMType, circuit_builder::{CircuitBuilder, ConstraintSystem}, error::ZKVMError, + instructions::InstancePaddingStrategy, scheme::mock_prover::MockProver, witness::{LkMultiplicity, RowMajorMatrix}, }; @@ -516,7 +515,11 @@ mod test { let mut lkm = LkMultiplicity::default(); let num_rows = 2; - let mut raw_witin = RowMajorMatrix::::new(num_rows, cb.cs.num_witin as usize); + let mut raw_witin = RowMajorMatrix::::new( + num_rows, + cb.cs.num_witin as usize, + InstancePaddingStrategy::Default, + ); for instance in raw_witin.iter_mut() { mem_addr.assign_instance(instance, &mut lkm, addr)?; } @@ -542,7 +545,6 @@ mod test { MockProver::assert_with_expected_errors( &cb, &raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) diff --git a/ceno_zkvm/src/instructions/riscv/j_insn.rs b/ceno_zkvm/src/instructions/riscv/j_insn.rs index 973493fdf..81cc8c493 100644 --- a/ceno_zkvm/src/instructions/riscv/j_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/j_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; // Opcode: 1101111 @@ -53,7 +52,7 @@ impl JInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/auipc.rs b/ceno_zkvm/src/instructions/riscv/jump/auipc.rs index 28be09c66..ed1f07d1f 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/auipc.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/auipc.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -71,7 +71,7 @@ impl Instruction for AuipcInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/jal.rs b/ceno_zkvm/src/instructions/riscv/jump/jal.rs index 44facf944..24bcad463 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/jal.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/jal.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff_ext::ExtensionField; @@ -62,7 +62,7 @@ impl Instruction for JalInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/jalr.rs b/ceno_zkvm/src/instructions/riscv/jump/jalr.rs index 0339a6b0a..1d9c2c025 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/jalr.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/jalr.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff::Field; use ff_ext::ExtensionField; @@ -110,7 +110,7 @@ impl Instruction for JalrInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/lui.rs b/ceno_zkvm/src/instructions/riscv/jump/lui.rs index 1ac67f64f..8ad9d497e 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/lui.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/lui.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -48,7 +48,7 @@ impl Instruction for LuiInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs b/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs index ae9235402..30fe370e0 100644 --- a/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs @@ -1,6 +1,5 @@ //! The circuit implementation of logic instructions. -use core::mem::MaybeUninit; use ff_ext::ExtensionField; use std::marker::PhantomData; @@ -50,7 +49,7 @@ impl Instruction for LogicInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -102,7 +101,7 @@ impl LogicConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs b/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs index b6a8bb690..9328e27a7 100644 --- a/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs @@ -1,6 +1,5 @@ //! The circuit implementation of logic instructions. -use core::mem::MaybeUninit; use ff_ext::ExtensionField; use std::marker::PhantomData; @@ -50,7 +49,7 @@ impl Instruction for LogicInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -103,7 +102,7 @@ impl LogicConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/memory/gadget.rs b/ceno_zkvm/src/instructions/riscv/memory/gadget.rs index 6539c1325..e43d862fe 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/gadget.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/gadget.rs @@ -11,7 +11,6 @@ use ceno_emul::StepRecord; use ff::Field; use ff_ext::ExtensionField; use itertools::izip; -use std::mem::MaybeUninit; pub struct MemWordChange { prev_limb_bytes: Vec, @@ -147,7 +146,7 @@ impl MemWordChange { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, shift: u32, diff --git a/ceno_zkvm/src/instructions/riscv/memory/load.rs b/ceno_zkvm/src/instructions/riscv/memory/load.rs index a06c17687..fff79f2df 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/load.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/load.rs @@ -19,7 +19,7 @@ use crate::{ use ceno_emul::{ByteAddr, InsnKind, StepRecord}; use ff_ext::ExtensionField; use itertools::izip; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct LoadConfig { im_insn: IMInstructionConfig, @@ -199,7 +199,7 @@ impl Instruction for LoadInstruction], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/memory/store.rs b/ceno_zkvm/src/instructions/riscv/memory/store.rs index d1d941b97..29c950957 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/store.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/store.rs @@ -17,7 +17,7 @@ use crate::{ }; use ceno_emul::{ByteAddr, InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct StoreConfig { s_insn: SInstructionConfig, @@ -128,7 +128,7 @@ impl Instruction fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/mul.rs b/ceno_zkvm/src/instructions/riscv/mul.rs index 16a08fe41..2214c437b 100644 --- a/ceno_zkvm/src/instructions/riscv/mul.rs +++ b/ceno_zkvm/src/instructions/riscv/mul.rs @@ -101,7 +101,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; pub struct MulhInstructionBase(PhantomData<(E, I)>); @@ -309,7 +308,7 @@ impl Instruction for MulhInstructionBas fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -423,7 +422,7 @@ impl Signed { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, val: &Value, ) -> Result { diff --git a/ceno_zkvm/src/instructions/riscv/r_insn.rs b/ceno_zkvm/src/instructions/riscv/r_insn.rs index 6e476a69a..78c262290 100644 --- a/ceno_zkvm/src/instructions/riscv/r_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/r_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of R-type instructions: /// - PC, cycle, fetch. @@ -61,7 +60,7 @@ impl RInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/s_insn.rs b/ceno_zkvm/src/instructions/riscv/s_insn.rs index dc133e894..efe271337 100644 --- a/ceno_zkvm/src/instructions/riscv/s_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/s_insn.rs @@ -9,7 +9,6 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; /// This config handles the common part of S-type instructions: /// - PC, cycle, fetch. @@ -70,7 +69,7 @@ impl SInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/shift.rs b/ceno_zkvm/src/instructions/riscv/shift.rs index 365afe786..e375912af 100644 --- a/ceno_zkvm/src/instructions/riscv/shift.rs +++ b/ceno_zkvm/src/instructions/riscv/shift.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -163,7 +163,7 @@ impl Instruction for ShiftLogicalInstru fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut crate::witness::LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), crate::error::ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/shift_imm.rs b/ceno_zkvm/src/instructions/riscv/shift_imm.rs index 080a8a6ae..50ebe4ff0 100644 --- a/ceno_zkvm/src/instructions/riscv/shift_imm.rs +++ b/ceno_zkvm/src/instructions/riscv/shift_imm.rs @@ -15,7 +15,7 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct ShiftImmConfig { i_insn: IInstructionConfig, @@ -144,7 +144,7 @@ impl Instruction for ShiftImmInstructio fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/slt.rs b/ceno_zkvm/src/instructions/riscv/slt.rs index 80fc69874..08ceff8b3 100644 --- a/ceno_zkvm/src/instructions/riscv/slt.rs +++ b/ceno_zkvm/src/instructions/riscv/slt.rs @@ -16,7 +16,6 @@ use crate::{ uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; pub struct SetLessThanInstruction(PhantomData<(E, I)>); @@ -101,7 +100,7 @@ impl Instruction for SetLessThanInstruc fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/slti.rs b/ceno_zkvm/src/instructions/riscv/slti.rs index 76894f7a0..a0766b425 100644 --- a/ceno_zkvm/src/instructions/riscv/slti.rs +++ b/ceno_zkvm/src/instructions/riscv/slti.rs @@ -20,7 +20,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; #[derive(Debug)] pub struct SetLessThanImmConfig { @@ -96,7 +95,7 @@ impl Instruction for SetLessThanImmInst fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/u_insn.rs b/ceno_zkvm/src/instructions/riscv/u_insn.rs index 8b00969f8..719ee2df1 100644 --- a/ceno_zkvm/src/instructions/riscv/u_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/u_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of the U-type instruction: /// - PC, cycle, fetch @@ -51,7 +50,7 @@ impl UInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/scheme/mock_prover.rs b/ceno_zkvm/src/scheme/mock_prover.rs index b84e94148..21b6d4c05 100644 --- a/ceno_zkvm/src/scheme/mock_prover.rs +++ b/ceno_zkvm/src/scheme/mock_prover.rs @@ -653,10 +653,7 @@ impl<'a, E: ExtensionField + Hash> MockProver { for table_expr in &cs.lk_table_expressions { for row in fixed.iter_rows() { // TODO: Find a better way to obtain the row content. - let row = row - .iter() - .map(|v| unsafe { (*v).assume_init() }.into()) - .collect::>(); + let row = row.iter().map(|v| (*v).into()).collect::>(); let rlc_record = eval_by_expr_with_fixed(&row, &[], &challenge, &table_expr.values); t_vec.push(rlc_record.to_canonical_u64_vec()); } @@ -727,7 +724,6 @@ Hints: lkm: Option, ) { let wits_in = raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) @@ -1206,7 +1202,6 @@ Hints: #[cfg(test)] mod tests { - use std::mem::MaybeUninit; use super::*; use crate::{ @@ -1214,6 +1209,7 @@ mod tests { error::ZKVMError, expression::{ToExpr, WitIn}, gadgets::{AssertLtConfig, IsLtConfig}, + instructions::InstancePaddingStrategy, set_val, witness::{LkMultiplicity, RowMajorMatrix}, }; @@ -1375,7 +1371,7 @@ mod tests { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], input: AssertLtCircuitInput, lk_multiplicity: &mut LkMultiplicity, ) -> Result<(), ZKVMError> { @@ -1393,7 +1389,11 @@ mod tests { instances: Vec, lk_multiplicity: &mut LkMultiplicity, ) -> Result, ZKVMError> { - let mut raw_witin = RowMajorMatrix::::new(instances.len(), num_witin); + let mut raw_witin = RowMajorMatrix::::new( + instances.len(), + num_witin, + InstancePaddingStrategy::Default, + ); let raw_witin_iter = raw_witin.iter_mut(); raw_witin_iter @@ -1489,7 +1489,7 @@ mod tests { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], input: LtCircuitInput, lk_multiplicity: &mut LkMultiplicity, ) -> Result<(), ZKVMError> { @@ -1507,7 +1507,11 @@ mod tests { instances: Vec, lk_multiplicity: &mut LkMultiplicity, ) -> Result, ZKVMError> { - let mut raw_witin = RowMajorMatrix::::new(instances.len(), num_witin); + let mut raw_witin = RowMajorMatrix::::new( + instances.len(), + num_witin, + InstancePaddingStrategy::Default, + ); let raw_witin_iter = raw_witin.iter_mut(); raw_witin_iter diff --git a/ceno_zkvm/src/scheme/tests.rs b/ceno_zkvm/src/scheme/tests.rs index 13ef29a66..945e49730 100644 --- a/ceno_zkvm/src/scheme/tests.rs +++ b/ceno_zkvm/src/scheme/tests.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ark_std::test_rng; use ceno_emul::{ @@ -77,7 +77,7 @@ impl Instruction for Test fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], _lk_multiplicity: &mut LkMultiplicity, _step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/tables/mod.rs b/ceno_zkvm/src/tables/mod.rs index 892babb3f..ef1a4fd68 100644 --- a/ceno_zkvm/src/tables/mod.rs +++ b/ceno_zkvm/src/tables/mod.rs @@ -1,12 +1,6 @@ -use crate::{ - circuit_builder::CircuitBuilder, error::ZKVMError, scheme::constants::MIN_PAR_SIZE, - witness::RowMajorMatrix, -}; -use ff::Field; +use crate::{circuit_builder::CircuitBuilder, error::ZKVMError, witness::RowMajorMatrix}; use ff_ext::ExtensionField; -use multilinear_extensions::util::max_usable_threads; -use rayon::iter::{IndexedParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; mod range; pub use range::*; @@ -42,29 +36,4 @@ pub trait TableCircuit { multiplicity: &[HashMap], input: &Self::WitnessInput, ) -> Result, ZKVMError>; - - fn padding_zero( - table: &mut RowMajorMatrix, - num_witin: usize, - ) -> Result<(), ZKVMError> { - // Fill the padding with zeros, if any. - let num_padding_instances = table.num_padding_instances(); - if num_padding_instances > 0 { - let nthreads = max_usable_threads(); - let padding_instance = vec![MaybeUninit::new(E::BaseField::ZERO); num_witin]; - let num_padding_instance_per_batch = if num_padding_instances > 256 { - num_padding_instances.div_ceil(nthreads) - } else { - num_padding_instances - }; - table - .par_batch_iter_padding_mut(num_padding_instance_per_batch) - .with_min_len(MIN_PAR_SIZE) - .for_each(|row| { - row.chunks_mut(num_witin) - .for_each(|instance| instance.copy_from_slice(padding_instance.as_slice())); - }); - } - Ok(()) - } } diff --git a/ceno_zkvm/src/tables/ops.rs b/ceno_zkvm/src/tables/ops.rs index 789f7f744..d6da1de33 100644 --- a/ceno_zkvm/src/tables/ops.rs +++ b/ceno_zkvm/src/tables/ops.rs @@ -124,12 +124,9 @@ mod tests { for (i, row) in fixed.iter_rows().enumerate() { let (base, exp) = PowTable::unpack(i as u64); assert_eq!(PowTable::pack(base, exp), i as u64); - assert_eq!(base, unsafe { row[0].assume_init() }.to_canonical_u64()); - assert_eq!(exp, unsafe { row[1].assume_init() }.to_canonical_u64()); - assert_eq!( - base.pow(exp.try_into().unwrap()), - unsafe { row[2].assume_init() }.to_canonical_u64() - ); + assert_eq!(base, row[0].to_canonical_u64()); + assert_eq!(exp, row[1].to_canonical_u64()); + assert_eq!(base.pow(exp.try_into().unwrap()), row[2].to_canonical_u64()); } } } diff --git a/ceno_zkvm/src/tables/ops/ops_circuit.rs b/ceno_zkvm/src/tables/ops/ops_circuit.rs index cd48ebe19..852702ea7 100644 --- a/ceno_zkvm/src/tables/ops/ops_circuit.rs +++ b/ceno_zkvm/src/tables/ops/ops_circuit.rs @@ -51,9 +51,7 @@ impl TableCircuit for OpsTableCircuit num_fixed: usize, _input: &(), ) -> RowMajorMatrix { - let mut table = config.generate_fixed_traces(num_fixed, OP::content()); - Self::padding_zero(&mut table, num_fixed).expect("padding error"); - table + config.generate_fixed_traces(num_fixed, OP::content()) } fn assign_instances( @@ -63,8 +61,6 @@ impl TableCircuit for OpsTableCircuit _input: &(), ) -> Result, ZKVMError> { let multiplicity = &multiplicity[OP::ROM_TYPE as usize]; - let mut table = config.assign_instances(num_witin, multiplicity, OP::len())?; - Self::padding_zero(&mut table, num_witin)?; - Ok(table) + config.assign_instances(num_witin, multiplicity, OP::len()) } } diff --git a/ceno_zkvm/src/tables/ops/ops_impl.rs b/ceno_zkvm/src/tables/ops/ops_impl.rs index ddd407078..3fe75d242 100644 --- a/ceno_zkvm/src/tables/ops/ops_impl.rs +++ b/ceno_zkvm/src/tables/ops/ops_impl.rs @@ -4,12 +4,13 @@ use ff_ext::ExtensionField; use goldilocks::SmallField; use itertools::Itertools; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, @@ -47,7 +48,8 @@ impl OpTableConfig { num_fixed: usize, content: Vec<[u64; 3]>, ) -> RowMajorMatrix { - let mut fixed = RowMajorMatrix::::new(content.len(), num_fixed); + let mut fixed = + RowMajorMatrix::::new(content.len(), num_fixed, InstancePaddingStrategy::Default); fixed .par_iter_mut() @@ -68,7 +70,8 @@ impl OpTableConfig { multiplicity: &HashMap, length: usize, ) -> Result, ZKVMError> { - let mut witness = RowMajorMatrix::::new(length, num_witin); + let mut witness = + RowMajorMatrix::::new(length, num_witin, InstancePaddingStrategy::Default); let mut mlts = vec![0; length]; for (idx, mlt) in multiplicity { diff --git a/ceno_zkvm/src/tables/program.rs b/ceno_zkvm/src/tables/program.rs index da063545e..a5ae2ad7f 100644 --- a/ceno_zkvm/src/tables/program.rs +++ b/ceno_zkvm/src/tables/program.rs @@ -1,9 +1,10 @@ -use std::{collections::HashMap, marker::PhantomData, mem::MaybeUninit}; +use std::{collections::HashMap, marker::PhantomData}; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, @@ -158,7 +159,11 @@ impl TableCircuit for ProgramTableCircuit { let pc_base = program.base_address; assert!(num_instructions <= config.program_size); - let mut fixed = RowMajorMatrix::::new(config.program_size, num_fixed); + let mut fixed = RowMajorMatrix::::new( + config.program_size, + num_fixed, + InstancePaddingStrategy::Default, + ); fixed .par_iter_mut() @@ -167,7 +172,7 @@ impl TableCircuit for ProgramTableCircuit { .for_each(|(row, i)| { let pc = pc_base + (i * PC_STEP_SIZE) as u32; let insn = DecodedInstruction::new(program.instructions[i]); - let values = InsnRecord::from_decoded(pc, &insn); + let values: InsnRecord<_> = InsnRecord::from_decoded(pc, &insn); // Copy all the fields. for (col, val) in config.record.as_slice().iter().zip_eq(values.as_slice()) { @@ -175,17 +180,6 @@ impl TableCircuit for ProgramTableCircuit { } }); - assert_eq!(INVALID as u64, 0, "0 padding must be invalid instructions"); - fixed - .par_iter_mut() - .with_min_len(MIN_PAR_SIZE) - .skip(num_instructions) - .for_each(|row| { - for col in config.record.as_slice() { - set_fixed_val!(row, *col, 0_u64.into()); - } - }); - fixed } @@ -203,7 +197,11 @@ impl TableCircuit for ProgramTableCircuit { prog_mlt[i] = *mlt; } - let mut witness = RowMajorMatrix::::new(config.program_size, num_witin); + let mut witness = RowMajorMatrix::::new( + config.program_size, + num_witin, + InstancePaddingStrategy::Default, + ); witness .par_iter_mut() .with_min_len(MIN_PAR_SIZE) @@ -212,14 +210,6 @@ impl TableCircuit for ProgramTableCircuit { set_val!(row, config.mlt, E::BaseField::from(mlt as u64)); }); - witness - .par_iter_mut() - .with_min_len(MIN_PAR_SIZE) - .skip(program.instructions.len()) - .for_each(|row| { - set_val!(row, config.mlt, 0_u64); - }); - Ok(witness) } } @@ -273,7 +263,7 @@ mod tests { ); for row in matrix.iter_rows().skip(actual_len) { for col in row.iter() { - assert_eq!(unsafe { col.assume_init() }, F::ZERO); + assert_eq!(*col, F::ZERO); } } }; diff --git a/ceno_zkvm/src/tables/ram/ram_circuit.rs b/ceno_zkvm/src/tables/ram/ram_circuit.rs index 234ff7799..3080f68bf 100644 --- a/ceno_zkvm/src/tables/ram/ram_circuit.rs +++ b/ceno_zkvm/src/tables/ram/ram_circuit.rs @@ -6,6 +6,7 @@ use ff_ext::ExtensionField; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, + instructions::InstancePaddingStrategy, structs::{ProgramParams, RAMType}, tables::TableCircuit, witness::RowMajorMatrix, @@ -204,7 +205,7 @@ impl TableC _num_fixed: usize, _init_v: &Self::FixedInput, ) -> RowMajorMatrix { - RowMajorMatrix::::new(0, 0) + RowMajorMatrix::::new(0, 0, InstancePaddingStrategy::Default) } fn assign_instances( diff --git a/ceno_zkvm/src/tables/ram/ram_impl.rs b/ceno_zkvm/src/tables/ram/ram_impl.rs index 2f16c6bff..8a31633bc 100644 --- a/ceno_zkvm/src/tables/ram/ram_impl.rs +++ b/ceno_zkvm/src/tables/ram/ram_impl.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::{marker::PhantomData, sync::Arc}; use ceno_emul::{Addr, Cycle}; use ff_ext::ExtensionField; @@ -10,7 +10,10 @@ use crate::{ circuit_builder::{CircuitBuilder, DynamicAddr, SetTableAddrType, SetTableSpec}, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, - instructions::riscv::constants::{LIMB_BITS, LIMB_MASK}, + instructions::{ + InstancePaddingStrategy, + riscv::constants::{LIMB_BITS, LIMB_MASK}, + }, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ProgramParams, @@ -116,7 +119,11 @@ impl NonVolatileTableConfig::new(NVRAM::len(&self.params), num_fixed); + let mut init_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_fixed, + InstancePaddingStrategy::Default, + ); assert_eq!(init_table.num_padding_instances(), 0); init_table @@ -146,7 +153,11 @@ impl NonVolatileTableConfig Result, ZKVMError> { - let mut final_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_witness); + let mut final_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_witness, + InstancePaddingStrategy::Default, + ); final_table .par_iter_mut() @@ -246,7 +257,11 @@ impl PubIOTableConfig { ) -> RowMajorMatrix { assert!(NVRAM::len(&self.params).is_power_of_two()); - let mut init_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_fixed); + let mut init_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_fixed, + InstancePaddingStrategy::Default, + ); assert_eq!(init_table.num_padding_instances(), 0); init_table @@ -265,7 +280,11 @@ impl PubIOTableConfig { num_witness: usize, final_cycles: &[Cycle], ) -> Result, ZKVMError> { - let mut final_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_witness); + let mut final_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_witness, + InstancePaddingStrategy::Default, + ); final_table .par_iter_mut() @@ -369,7 +388,22 @@ impl DynVolatileRamTableConfig ) -> Result, ZKVMError> { assert!(final_mem.len() <= DVRAM::max_len(&self.params)); assert!(DVRAM::max_len(&self.params).is_power_of_two()); - let mut final_table = RowMajorMatrix::::new(final_mem.len(), num_witness); + + let params = self.params.clone(); + let addr_column = self.addr.id as u64; + let padding_fn = move |row: u64, col: u64| { + if col == addr_column { + DVRAM::addr(¶ms, row as usize) as u64 + } else { + 0u64 + } + }; + + let mut final_table = RowMajorMatrix::::new( + final_mem.len(), + num_witness, + InstancePaddingStrategy::Custom(Arc::new(padding_fn)), + ); final_table .par_iter_mut() @@ -393,21 +427,63 @@ impl DynVolatileRamTableConfig set_val!(row, self.final_cycle, rec.cycle); }); - // set padding with well-form address - final_table - .par_iter_mut() - .enumerate() - .skip(final_mem.len()) - .with_min_len(MIN_PAR_SIZE) - .for_each(|(i, row)| { - // Assign value limbs. - self.final_v.iter().for_each(|limb| { - set_val!(row, limb, 0u64); - }); - set_val!(row, self.addr, DVRAM::addr(&self.params, i) as u64); - set_val!(row, self.final_cycle, 0_u64); - }); - Ok(final_table) } } + +#[cfg(test)] +mod tests { + use std::iter::successors; + + use crate::{ + circuit_builder::{CircuitBuilder, ConstraintSystem}, + structs::ProgramParams, + tables::{DynVolatileRamTable, HintsCircuit, HintsTable, MemFinalRecord, TableCircuit}, + utils::next_pow2_instance_padding, + witness::LkMultiplicity, + }; + + use ceno_emul::WORD_SIZE; + use goldilocks::{Goldilocks as F, GoldilocksExt2 as E}; + use itertools::Itertools; + + #[test] + fn test_well_formed_address_padding() { + let mut cs = ConstraintSystem::::new(|| "riscv"); + let mut cb = CircuitBuilder::new(&mut cs); + let config = HintsCircuit::construct_circuit(&mut cb).unwrap(); + + let def_params = ProgramParams::default(); + let lkm = LkMultiplicity::default().into_finalize_result(); + + // ensure non-empty padding is required + let some_non_2_pow = 26; + let input = (0..some_non_2_pow) + .map(|i| MemFinalRecord { + addr: HintsTable::addr(&def_params, i), + cycle: 0, + value: 0, + }) + .collect_vec(); + let wit = + HintsCircuit::::assign_instances(&config, cb.cs.num_witin as usize, &lkm, &input) + .unwrap(); + + let addr_column = cb + .cs + .witin_namespace_map + .iter() + .position(|name| name == "riscv/RAM_Memory_HintsTable/addr") + .unwrap(); + + let addr_padded_view = wit.column_padded(addr_column); + // Expect addresses to proceed consecutively inside the padding as well + let expected = successors(Some(addr_padded_view[0]), |idx| { + Some(*idx + F::from(WORD_SIZE as u64)) + }) + .take(next_pow2_instance_padding(wit.num_instances())) + .collect::>(); + + assert_eq!(addr_padded_view, expected) + } +} diff --git a/ceno_zkvm/src/tables/range/range_circuit.rs b/ceno_zkvm/src/tables/range/range_circuit.rs index bb7c83448..d5fdf7363 100644 --- a/ceno_zkvm/src/tables/range/range_circuit.rs +++ b/ceno_zkvm/src/tables/range/range_circuit.rs @@ -44,9 +44,7 @@ impl TableCircuit for RangeTableCircuit num_fixed: usize, _input: &(), ) -> RowMajorMatrix { - let mut table = config.generate_fixed_traces(num_fixed, RANGE::content()); - Self::padding_zero(&mut table, num_fixed).expect("padding error"); - table + config.generate_fixed_traces(num_fixed, RANGE::content()) } fn assign_instances( @@ -56,8 +54,6 @@ impl TableCircuit for RangeTableCircuit _input: &(), ) -> Result, ZKVMError> { let multiplicity = &multiplicity[RANGE::ROM_TYPE as usize]; - let mut table = config.assign_instances(num_witin, multiplicity, RANGE::len())?; - Self::padding_zero(&mut table, num_witin).expect("padding error"); - Ok(table) + config.assign_instances(num_witin, multiplicity, RANGE::len()) } } diff --git a/ceno_zkvm/src/tables/range/range_impl.rs b/ceno_zkvm/src/tables/range/range_impl.rs index 505ef4b4f..30937824c 100644 --- a/ceno_zkvm/src/tables/range/range_impl.rs +++ b/ceno_zkvm/src/tables/range/range_impl.rs @@ -3,12 +3,13 @@ use ff_ext::ExtensionField; use goldilocks::SmallField; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, @@ -42,7 +43,8 @@ impl RangeTableConfig { num_fixed: usize, content: Vec, ) -> RowMajorMatrix { - let mut fixed = RowMajorMatrix::::new(content.len(), num_fixed); + let mut fixed = + RowMajorMatrix::::new(content.len(), num_fixed, InstancePaddingStrategy::Default); fixed .par_iter_mut() @@ -61,7 +63,8 @@ impl RangeTableConfig { multiplicity: &HashMap, length: usize, ) -> Result, ZKVMError> { - let mut witness = RowMajorMatrix::::new(length, num_witin); + let mut witness = + RowMajorMatrix::::new(length, num_witin, InstancePaddingStrategy::Default); let mut mlts = vec![0; length]; for (idx, mlt) in multiplicity { diff --git a/ceno_zkvm/src/uint.rs b/ceno_zkvm/src/uint.rs index b555682ac..b11490395 100644 --- a/ceno_zkvm/src/uint.rs +++ b/ceno_zkvm/src/uint.rs @@ -20,7 +20,7 @@ use goldilocks::SmallField; use itertools::{Itertools, enumerate}; use std::{ borrow::Cow, - mem::{self, MaybeUninit}, + mem::{self}, ops::Index, }; pub use strum::IntoEnumIterator; @@ -202,20 +202,20 @@ impl UIntLimbs { pub fn assign_value + Default + From + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], value: Value, ) { self.assign_limbs(instance, value.as_u16_limbs()) } - pub fn assign_add_outcome(&self, instance: &mut [MaybeUninit], value: &ValueAdd) { + pub fn assign_add_outcome(&self, instance: &mut [E::BaseField], value: &ValueAdd) { self.assign_limbs(instance, &value.limbs); self.assign_carries(instance, &value.carries); } pub fn assign_mul_outcome( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, value: &ValueMul, ) -> Result<(), ZKVMError> { @@ -224,7 +224,7 @@ impl UIntLimbs { self.assign_carries_auxiliary(instance, lkm, &value.carries, value.max_carry_value) } - pub fn assign_limbs(&self, instance: &mut [MaybeUninit], limbs_values: &[u16]) { + pub fn assign_limbs(&self, instance: &mut [E::BaseField], limbs_values: &[u16]) { assert!( limbs_values.len() <= Self::NUM_LIMBS, "assign input length mismatch. input_len={}, NUM_CELLS={}", @@ -238,14 +238,14 @@ impl UIntLimbs { .map(|v| E::BaseField::from(*v as u64)) .chain(std::iter::repeat(E::BaseField::ZERO)), ) { - instance[wire.id as usize] = MaybeUninit::new(limb); + instance[wire.id as usize] = limb; } } } pub fn assign_carries + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], carry_values: &[T], ) { assert!( @@ -264,14 +264,14 @@ impl UIntLimbs { .map(|v| E::BaseField::from(Into::::into(*v))) .chain(std::iter::repeat(E::BaseField::ZERO)), ) { - instance[wire.id as usize] = MaybeUninit::new(carry); + instance[wire.id as usize] = carry; } } } pub fn assign_carries_auxiliary + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, carry_values: &[T], max_carry: u64, diff --git a/ceno_zkvm/src/witness.rs b/ceno_zkvm/src/witness.rs index 7acc9ad50..97150e086 100644 --- a/ceno_zkvm/src/witness.rs +++ b/ceno_zkvm/src/witness.rs @@ -3,23 +3,21 @@ use std::{ array, cell::RefCell, collections::HashMap, - mem::{self, MaybeUninit}, + mem::{self}, ops::Index, slice::{Chunks, ChunksMut}, sync::Arc, }; -use multilinear_extensions::{ - mle::{DenseMultilinearExtension, IntoMLEs}, - util::create_uninit_vec, -}; +use multilinear_extensions::mle::{DenseMultilinearExtension, IntoMLE}; use rayon::{ - iter::{IndexedParallelIterator, IntoParallelRefMutIterator, ParallelIterator}, + iter::{IntoParallelIterator, ParallelIterator}, slice::ParallelSliceMut, }; use thread_local::ThreadLocal; use crate::{ + instructions::InstancePaddingStrategy, structs::ROMType, tables::{AndTable, LtuTable, OpsTable, OrTable, PowTable, XorTable}, utils::next_pow2_instance_padding, @@ -28,97 +26,99 @@ use crate::{ #[macro_export] macro_rules! set_val { ($ins:ident, $field:expr, $val:expr) => { - $ins[$field.id as usize] = MaybeUninit::new($val.into()); + $ins[$field.id as usize] = $val.into(); }; } #[macro_export] macro_rules! set_fixed_val { ($ins:ident, $field:expr, $val:expr) => { - $ins[$field.0] = MaybeUninit::new($val); + $ins[$field.0] = $val; }; } #[derive(Clone)] pub struct RowMajorMatrix { // represent 2D in 1D linear memory and avoid double indirection by Vec> to improve performance - values: Vec>, - num_padding_rows: usize, + values: Vec, num_col: usize, + padding_strategy: InstancePaddingStrategy, } -impl RowMajorMatrix { - pub fn new(num_rows: usize, num_col: usize) -> Self { - let num_total_rows = next_pow2_instance_padding(num_rows); - let num_padding_rows = num_total_rows - num_rows; +impl> RowMajorMatrix { + pub fn new(num_rows: usize, num_col: usize, padding_strategy: InstancePaddingStrategy) -> Self { RowMajorMatrix { - values: create_uninit_vec(num_total_rows * num_col), - num_padding_rows, + values: (0..num_rows * num_col) + .into_par_iter() + .map(|_| T::default()) + .collect(), num_col, + padding_strategy, } } - pub fn num_instances(&self) -> usize { - self.values.len() / self.num_col - self.num_padding_rows + pub fn num_padding_instances(&self) -> usize { + next_pow2_instance_padding(self.num_instances()) - self.num_instances() } - pub fn num_padding_instances(&self) -> usize { - self.num_padding_rows + pub fn num_instances(&self) -> usize { + self.values.len() / self.num_col } - pub fn iter_rows(&self) -> Chunks> { + pub fn iter_rows(&self) -> Chunks { self.values.chunks(self.num_col) } - pub fn iter_mut(&mut self) -> ChunksMut> { + pub fn iter_mut(&mut self) -> ChunksMut { self.values.chunks_mut(self.num_col) } - pub fn par_iter_mut(&mut self) -> rayon::slice::ChunksMut> { + pub fn par_iter_mut(&mut self) -> rayon::slice::ChunksMut { self.values.par_chunks_mut(self.num_col) } - pub fn par_batch_iter_mut( - &mut self, - num_rows: usize, - ) -> rayon::slice::ChunksMut> { + pub fn par_batch_iter_mut(&mut self, num_rows: usize) -> rayon::slice::ChunksMut { self.values.par_chunks_mut(num_rows * self.num_col) } - pub fn par_batch_iter_padding_mut( - &mut self, - num_rows: usize, - ) -> rayon::slice::ChunksMut<'_, MaybeUninit> { - let valid_instance = self.num_instances(); - self.values[valid_instance * self.num_col..] - .as_mut() - .par_chunks_mut(num_rows * self.num_col) - } - - pub fn de_interleaving(mut self) -> Vec> { - (0..self.num_col) - .map(|i| { - self.values - .par_iter_mut() - .skip(i) - .step_by(self.num_col) - .map(|v| unsafe { mem::replace(v, mem::MaybeUninit::uninit()).assume_init() }) - .collect::>() - }) - .collect() + // Returns column number `column`, padded appropriately according to the stored strategy + pub fn column_padded(&self, column: usize) -> Vec { + let num_instances = self.num_instances(); + let num_padding_instances = self.num_padding_instances(); + + let padding_iter = (num_instances..num_instances + num_padding_instances).map(|i| { + match &self.padding_strategy { + InstancePaddingStrategy::Custom(fun) => T::from(fun(i as u64, column as u64)), + InstancePaddingStrategy::RepeatLast if num_instances > 0 => { + self[num_instances - 1][column] + } + _ => T::default(), + } + }); + + self.values + .iter() + .skip(column) + .step_by(self.num_col) + .copied() + .chain(padding_iter) + .collect::>() } } -impl RowMajorMatrix { +impl> RowMajorMatrix { pub fn into_mles>( self, ) -> Vec> { - self.de_interleaving().into_mles() + (0..self.num_col) + .into_par_iter() + .map(|i| self.column_padded(i).into_mle()) + .collect() } } -impl Index for RowMajorMatrix { - type Output = [MaybeUninit]; +impl Index for RowMajorMatrix { + type Output = [F]; fn index(&self, idx: usize) -> &Self::Output { &self.values[self.num_col * idx..][..self.num_col] diff --git a/multilinear_extensions/src/mle.rs b/multilinear_extensions/src/mle.rs index 8a182e645..b4e8df983 100644 --- a/multilinear_extensions/src/mle.rs +++ b/multilinear_extensions/src/mle.rs @@ -103,9 +103,9 @@ pub trait IntoMLE: Sized { } impl IntoMLE> for Vec { - fn into_mle(mut self) -> DenseMultilinearExtension { + fn into_mle(self) -> DenseMultilinearExtension { let next_pow2 = self.len().next_power_of_two(); - self.resize(next_pow2, F::ZERO); + assert!(self.len().is_power_of_two(), "{}", self.len()); DenseMultilinearExtension::from_evaluation_vec_smart::(ceil_log2(next_pow2), self) } }