feat: impl marco morsel_record

tonbo-io · Jul 25, 2024 · db46a0c · db46a0c
1 parent a96d4b9
commit db46a0c
Show file tree

Hide file tree

Showing 11 changed files with 650 additions and 308 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,3 +1,5 @@
+workspace = { members = [ "src/morseldb_marco"] }
+
 [package]
 edition = "2021"
 name = "morseldb"
@@ -17,6 +19,7 @@ futures-core = "0.3"
 futures-io = "0.3"
 futures-util = "0.3"
 lockable = "0.0"
+morseldb_marco = { path = "src/morseldb_marco" }
 once_cell = "1"
 parquet = { version = "52", features = ["async"] }
 pin-project-lite = "0.2"

diff --git a/src/compaction/mod.rs b/src/compaction/mod.rs
@@ -474,23 +474,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 3.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 5.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 6.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
@@ -500,23 +500,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 4.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 2.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 1.to_string(),
                     vu32: 0,
-                    vobool: None,
+                    vbool: None,
                 },
                 0.into(),
             );
@@ -600,23 +600,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 1.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 1.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 2.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 1.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 3.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
@@ -628,23 +628,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 4.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 1.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 5.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 1.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 6.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
@@ -661,23 +661,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 1.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 2.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 3.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
@@ -689,23 +689,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 4.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 5.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 6.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
@@ -717,23 +717,23 @@ pub(crate) mod tests {
                 Test {
                     vstring: 7.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 8.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );
             mutable.insert(
                 Test {
                     vstring: 9.to_string(),
                     vu32: 0,
-                    vobool: Some(true),
+                    vbool: Some(true),
                 },
                 0.into(),
             );

diff --git a/src/inmem/immutable.rs b/src/inmem/immutable.rs
@@ -175,145 +175,3 @@ where
         })
     }
 }
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use std::sync::Arc;
-
-    use arrow::{
-        array::{
-            Array, BooleanArray, BooleanBufferBuilder, BooleanBuilder, PrimitiveBuilder,
-            RecordBatch, StringArray, StringBuilder, UInt32Array, UInt32Builder,
-        },
-        datatypes::{ArrowPrimitiveType, UInt32Type},
-    };
-    use parquet::arrow::ProjectionMask;
-
-    use super::{ArrowArrays, Builder};
-    use crate::{
-        record::Record,
-        tests::{Test, TestRef},
-        timestamp::timestamped::Timestamped,
-    };
-
-    #[derive(Debug)]
-    pub struct TestImmutableArrays {
-        _null: Arc<BooleanArray>,
-        _ts: Arc<UInt32Array>,
-        vstring: Arc<StringArray>,
-        vu32: Arc<UInt32Array>,
-        vbool: Arc<BooleanArray>,
-
-        record_batch: RecordBatch,
-    }
-
-    impl ArrowArrays for TestImmutableArrays {
-        type Record = Test;
-
-        type Builder = TestBuilder;
-
-        fn builder(capacity: usize) -> Self::Builder {
-            TestBuilder {
-                vstring: StringBuilder::with_capacity(capacity, 0),
-                vu32: PrimitiveBuilder::<UInt32Type>::with_capacity(capacity),
-                vobool: BooleanBuilder::with_capacity(capacity),
-                _null: BooleanBufferBuilder::new(capacity),
-                _ts: UInt32Builder::with_capacity(capacity),
-            }
-        }
-
-        fn get(
-            &self,
-            offset: u32,
-            projection_mask: &ProjectionMask,
-        ) -> Option<Option<<Self::Record as Record>::Ref<'_>>> {
-            let offset = offset as usize;
-
-            if offset >= self.vstring.len() {
-                return None;
-            }
-            if self._null.value(offset) {
-                return Some(None);
-            }
-
-            let vstring = self.vstring.value(offset);
-            let vu32 = projection_mask
-                .leaf_included(3)
-                .then(|| self.vu32.value(offset));
-            let vbool = (!self.vbool.is_null(offset) && projection_mask.leaf_included(4))
-                .then(|| self.vbool.value(offset));
-
-            Some(Some(TestRef {
-                vstring,
-                vu32,
-                vbool,
-            }))
-        }
-
-        fn as_record_batch(&self) -> &RecordBatch {
-            &self.record_batch
-        }
-    }
-
-    pub struct TestBuilder {
-        vstring: StringBuilder,
-        vu32: PrimitiveBuilder<UInt32Type>,
-        vobool: BooleanBuilder,
-        _null: BooleanBufferBuilder,
-        _ts: UInt32Builder,
-    }
-
-    impl Builder<TestImmutableArrays> for TestBuilder {
-        fn push(&mut self, key: Timestamped<&str>, row: Option<TestRef>) {
-            self.vstring.append_value(key.value);
-            match row {
-                Some(row) => {
-                    self.vu32.append_value(row.vu32.unwrap());
-                    match row.vbool {
-                        Some(vobool) => self.vobool.append_value(vobool),
-                        None => self.vobool.append_null(),
-                    }
-                    self._null.append(false);
-                    self._ts.append_value(key.ts.into());
-                }
-                None => {
-                    self.vu32
-                        .append_value(<UInt32Type as ArrowPrimitiveType>::Native::default());
-                    self.vobool.append_null();
-                    self._null.append(true);
-                    self._ts.append_value(key.ts.into());
-                }
-            }
-        }
-
-        fn finish(&mut self) -> TestImmutableArrays {
-            let vstring = Arc::new(self.vstring.finish());
-            let vu32 = Arc::new(self.vu32.finish());
-            let vbool = Arc::new(self.vobool.finish());
-            let _null = Arc::new(BooleanArray::new(self._null.finish(), None));
-            let _ts = Arc::new(self._ts.finish());
-            let record_batch = RecordBatch::try_new(
-                Arc::clone(
-                    <<TestImmutableArrays as ArrowArrays>::Record as Record>::arrow_schema(),
-                ),
-                vec![
-                    Arc::clone(&_null) as Arc<dyn Array>,
-                    Arc::clone(&_ts) as Arc<dyn Array>,
-                    Arc::clone(&vstring) as Arc<dyn Array>,
-                    Arc::clone(&vu32) as Arc<dyn Array>,
-                    Arc::clone(&vbool) as Arc<dyn Array>,
-                ],
-            )
-            .expect("create record batch must be successful");
-
-            TestImmutableArrays {
-                vstring,
-                vu32,
-                vbool,
-                _null,
-                _ts,
-                record_batch,
-            }
-        }
-    }
-}
diff --git a/src/inmem/mutable.rs b/src/inmem/mutable.rs
@@ -143,15 +143,15 @@ mod tests {
             Test {
                 vstring: key_1.clone(),
                 vu32: 1,
-                vobool: Some(true),
+                vbool: Some(true),
             },
             0_u32.into(),
         );
         mem_table.insert(
             Test {
                 vstring: key_2.clone(),
                 vu32: 2,
-                vobool: None,
+                vbool: None,
             },
             1_u32.into(),
         );