chore(mito): change default batch size/row group size (#2550)

GreptimeTeam · Oct 9, 2023 · 81aa7a4 · 81aa7a4
1 parent d68dd1f
commit 81aa7a4
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs
@@ -26,7 +26,10 @@ use crate::sst::file::FileTimeRange;
 /// Key of metadata in parquet SST.
 pub const PARQUET_METADATA_KEY: &str = "greptime:metadata";
 const DEFAULT_WRITE_BUFFER_SIZE: ReadableSize = ReadableSize::mb(8);
-const DEFAULT_ROW_GROUP_SIZE: usize = 100000;
+/// Default batch size to read parquet files.
+pub(crate) const DEFAULT_READ_BATCH_SIZE: usize = 1024;
+/// Default row group size for parquet files.
+const DEFAULT_ROW_GROUP_SIZE: usize = 100 * DEFAULT_READ_BATCH_SIZE;
 
 /// Parquet write options.
 #[derive(Debug)]

diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs
@@ -46,7 +46,7 @@ use crate::read::{Batch, BatchReader};
 use crate::sst::file::{FileHandle, FileId};
 use crate::sst::parquet::format::ReadFormat;
 use crate::sst::parquet::stats::RowGroupPruningStats;
-use crate::sst::parquet::PARQUET_METADATA_KEY;
+use crate::sst::parquet::{DEFAULT_READ_BATCH_SIZE, PARQUET_METADATA_KEY};
 
 /// Parquet SST reader builder.
 pub struct ParquetReaderBuilder {
@@ -147,7 +147,8 @@ impl ParquetReaderBuilder {
         };
         let mut builder = ParquetRecordBatchStreamBuilder::new(reader)
             .await
-            .context(ReadParquetSnafu { path: file_path })?;
+            .context(ReadParquetSnafu { path: file_path })?
+            .with_batch_size(DEFAULT_READ_BATCH_SIZE);
 
         // Decode region metadata.
         let key_value_meta = builder.metadata().file_metadata().key_value_metadata();