fix: written_size -> Builder::written_size

src/compaction/mod.rs

@@ -19,7 +19,6 @@ use crate::{
     ondisk::sstable::SsTable,
     record::{KeyRef, Record},
     scope::Scope,
-    serdes::Encode,
     stream::{level::LevelStream, merge::MergeStream, ScanStream},
     version::{
         edit::VersionEdit,
@@ -317,7 +316,6 @@ where
 
         // Kould: is the capacity parameter necessary?
         let mut builder = R::Columns::builder(8192);
-        let mut written_size = 0;
         let mut min = None;
         let mut max = None;
 
@@ -329,11 +327,9 @@ where
                 min = Some(key.value.to_key())
             }
             max = Some(key.value.to_key());
-
-            written_size += key.size() + entry.value().size();
             builder.push(key, entry.value());
 
-            if written_size >= option.max_sst_file_size {
+            if builder.written_size() >= option.max_sst_file_size {
                 Self::build_table(
                     option,
                     version_edits,
@@ -343,10 +339,9 @@ where
                     &mut max,
                 )
                 .await?;
-                written_size = 0;
             }
         }
-        if written_size > 0 {
+        if builder.written_size() > 0 {
             Self::build_table(
                 option,
                 version_edits,

src/inmem/immutable.rs

@@ -40,6 +40,8 @@ where
         row: Option<<S::Record as Record>::Ref<'_>>,
     );
 
+    fn written_size(&self) -> usize;
+
     fn finish(&mut self) -> S;
 }
 

src/lib.rs

@@ -299,7 +299,7 @@ pub enum Projection {
 
 #[cfg(test)]
 pub(crate) mod tests {
-    use std::{collections::VecDeque, sync::Arc};
+    use std::{collections::VecDeque, mem::size_of, sync::Arc};
 
     use arrow::{
         array::{

src/morseldb_marco/src/lib.rs

@@ -50,6 +50,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
     let mut builder_push_some_fields: Vec<proc_macro2::TokenStream> = Vec::new();
     // only normal fields
     let mut builder_push_none_fields: Vec<proc_macro2::TokenStream> = Vec::new();
+    let mut buidler_size_fields: Vec<proc_macro2::TokenStream> = Vec::new();
 
     if let Data::Struct(data_struct) = &ast.data {
         if let Fields::Named(fields) = &data_struct.fields {
@@ -69,6 +70,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                     builder_with_capacity_method,
                     builder,
                     default,
+                    size_method,
                 ) = match to_data_type(&field.ty) {
                     Some((DataType::UInt8, is_nullable)) => (
                         is_nullable,
@@ -79,6 +81,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<UInt8Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<UInt8Type>),
                         quote!(0),
+                        quote!(values_slice().len()),
                     ),
                     Some((DataType::UInt16, is_nullable)) => (
                         is_nullable,
@@ -89,6 +92,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<UInt16Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<UInt16Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<u16>()),
                     ),
                     Some((DataType::UInt32, is_nullable)) => (
                         is_nullable,
@@ -99,6 +103,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<UInt32Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<UInt32Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<u32>()),
                     ),
                     Some((DataType::UInt64, is_nullable)) => (
                         is_nullable,
@@ -109,6 +114,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<UInt64Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<UInt64Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<u64>()),
                     ),
 
                     Some((DataType::Int8, is_nullable)) => (
@@ -120,6 +126,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<Int8Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<Int8Type>),
                         quote!(0),
+                        quote!(values_slice().len()),
                     ),
                     Some((DataType::Int16, is_nullable)) => (
                         is_nullable,
@@ -130,6 +137,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<Int16Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<Int16Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<i16>()),
                     ),
                     Some((DataType::Int32, is_nullable)) => (
                         is_nullable,
@@ -140,6 +148,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<Int32Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<Int32Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<i32>()),
                     ),
                     Some((DataType::Int64, is_nullable)) => (
                         is_nullable,
@@ -150,6 +159,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(PrimitiveBuilder::<Int64Type>::with_capacity(capacity)),
                         quote!(PrimitiveBuilder<Int64Type>),
                         quote!(0),
+                        quote!(values_slice().len() * size_of::<i64>()),
                     ),
 
                     Some((DataType::String, is_nullable)) => {
@@ -163,6 +173,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                             quote!(StringBuilder::with_capacity(capacity, 0)),
                             quote!(StringBuilder),
                             quote!(""),
+                            quote!(values_slice().len()),
                         )
                     }
                     Some((DataType::Boolean, is_nullable)) => (
@@ -174,6 +185,7 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                         quote!(BooleanBuilder::with_capacity(capacity)),
                         quote!(BooleanBuilder),
                         quote!(false),
+                        quote!(values_slice().len()),
                     ),
 
                     None => unreachable!(),
@@ -204,6 +216,9 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                 encode_size_fields.push(quote! {
                     + self.#field_name.size()
                 });
+                buidler_size_fields.push(quote! {
+                    + self.#field_name.#size_method
+                });
 
                 match ModelAttributes::parse_field(field) {
                     Ok(false) => {
@@ -534,6 +549,10 @@ pub fn morsel_record(_args: TokenStream, input: TokenStream) -> TokenStream {
                 }
             }
 
+            fn written_size(&self) -> usize {
+                0 #(#buidler_size_fields)*
+            }
+
             fn finish(&mut self) -> #struct_arrays_name {
                 #(#builder_finish_fields)*
 

src/record/str.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{mem::size_of, sync::Arc};
 
 use arrow::{
     array::{
@@ -151,6 +151,12 @@ impl Builder<StringColumns> for StringColumnsBuilder {
         }
     }
 
+    fn written_size(&self) -> usize {
+        self._null.as_slice().len()
+            + self._ts.values_slice().len() * size_of::<u32>()
+            + self.string.values_slice().len()
+    }
+
     fn finish(&mut self) -> StringColumns {
         let _null = Arc::new(BooleanArray::new(self._null.finish(), None));
         let _ts = Arc::new(self._ts.finish());