From e178ae9b0f5cfb1d3e897baea3b630b5b800a5b5 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Thu, 26 Dec 2024 03:39:52 +0000 Subject: [PATCH] address comments Signed-off-by: Zhenchi --- config/datanode.example.toml | 18 +++++++++--------- config/standalone.example.toml | 2 +- src/mito2/src/access_layer.rs | 4 ++-- src/mito2/src/cache/write_cache.rs | 6 +++--- src/mito2/src/compaction/compactor.rs | 5 +++-- src/mito2/src/config.rs | 6 +++--- src/mito2/src/flush.rs | 2 +- src/mito2/src/sst/index.rs | 24 ++++++++++++------------ tests-integration/tests/http.rs | 2 +- 9 files changed, 35 insertions(+), 34 deletions(-) diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 5c7919d70325..6013c9a5c39a 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -576,37 +576,37 @@ apply_on_query = "auto" ## - `[size]` e.g. `64MB`: fixed memory threshold mem_threshold_on_create = "auto" -## The options for bloom filter in Mito engine. -[region_engine.mito.bloom_filter] +## The options for bloom filter index in Mito engine. +[region_engine.mito.bloom_filter_index] -## Whether to create the bloom filter on flush. +## Whether to create the index on flush. ## - `auto`: automatically (default) ## - `disable`: never create_on_flush = "auto" -## Whether to create the bloom filter on compaction. +## Whether to create the index on compaction. ## - `auto`: automatically (default) ## - `disable`: never create_on_compaction = "auto" -## Whether to apply the bloom filter on query +## Whether to apply the index on query ## - `auto`: automatically (default) ## - `disable`: never apply_on_query = "auto" -## Memory threshold for bloom filter creation. +## Memory threshold for the index creation. ## - `auto`: automatically determine the threshold based on the system memory size (default) ## - `unlimited`: no memory limit ## - `[size]` e.g. `64MB`: fixed memory threshold mem_threshold_on_create = "auto" -## Cache size for bloom filter metadata. +## Cache size for the index metadata. metadata_cache_size = "4MiB" -## Cache size for bloom filter content. +## Cache size for the index content. content_cache_size = "128MiB" -## Page size for bloom filter content cache. +## Page size for the index content cache. content_cache_page_size = "8MiB" [region_engine.mito.memtable] diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 52fb74b24197..763045351c30 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -620,7 +620,7 @@ apply_on_query = "auto" mem_threshold_on_create = "auto" ## The options for bloom filter in Mito engine. -[region_engine.mito.bloom_filter] +[region_engine.mito.bloom_filter_index] ## Whether to create the bloom filter on flush. ## - `auto`: automatically (default) diff --git a/src/mito2/src/access_layer.rs b/src/mito2/src/access_layer.rs index 7ea5221bd425..0d6204d02416 100644 --- a/src/mito2/src/access_layer.rs +++ b/src/mito2/src/access_layer.rs @@ -154,7 +154,7 @@ impl AccessLayer { index_options: request.index_options, inverted_index_config: request.inverted_index_config, fulltext_index_config: request.fulltext_index_config, - bloom_filter_config: request.bloom_filter_config, + bloom_filter_index_config: request.bloom_filter_index_config, } .build() .await; @@ -199,7 +199,7 @@ pub(crate) struct SstWriteRequest { pub(crate) index_options: IndexOptions, pub(crate) inverted_index_config: InvertedIndexConfig, pub(crate) fulltext_index_config: FulltextIndexConfig, - pub(crate) bloom_filter_config: BloomFilterConfig, + pub(crate) bloom_filter_index_config: BloomFilterConfig, } pub(crate) async fn new_fs_cache_store(root: &str) -> Result { diff --git a/src/mito2/src/cache/write_cache.rs b/src/mito2/src/cache/write_cache.rs index 8fc3db79c7e9..18fe41c5f614 100644 --- a/src/mito2/src/cache/write_cache.rs +++ b/src/mito2/src/cache/write_cache.rs @@ -125,7 +125,7 @@ impl WriteCache { index_options: write_request.index_options, inverted_index_config: write_request.inverted_index_config, fulltext_index_config: write_request.fulltext_index_config, - bloom_filter_config: write_request.bloom_filter_config, + bloom_filter_index_config: write_request.bloom_filter_index_config, } .build() .await; @@ -379,7 +379,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: Default::default(), fulltext_index_config: Default::default(), - bloom_filter_config: Default::default(), + bloom_filter_index_config: Default::default(), }; let upload_request = SstUploadRequest { @@ -472,7 +472,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: Default::default(), fulltext_index_config: Default::default(), - bloom_filter_config: Default::default(), + bloom_filter_index_config: Default::default(), }; let write_opts = WriteOptions { row_group_size: 512, diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs index 15503ce3cbb0..58425f4d79e3 100644 --- a/src/mito2/src/compaction/compactor.rs +++ b/src/mito2/src/compaction/compactor.rs @@ -301,7 +301,8 @@ impl Compactor for DefaultCompactor { let merge_mode = compaction_region.current_version.options.merge_mode(); let inverted_index_config = compaction_region.engine_config.inverted_index.clone(); let fulltext_index_config = compaction_region.engine_config.fulltext_index.clone(); - let bloom_filter_config = compaction_region.engine_config.bloom_filter.clone(); + let bloom_filter_index_config = + compaction_region.engine_config.bloom_filter_index.clone(); futs.push(async move { let reader = CompactionSstReaderBuilder { metadata: region_metadata.clone(), @@ -326,7 +327,7 @@ impl Compactor for DefaultCompactor { index_options, inverted_index_config, fulltext_index_config, - bloom_filter_config, + bloom_filter_index_config, }, &write_opts, ) diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index cdb5cad75ac6..400a5ecf85c6 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -117,8 +117,8 @@ pub struct MitoConfig { pub inverted_index: InvertedIndexConfig, /// Full-text index configs. pub fulltext_index: FulltextIndexConfig, - /// Bloom filter configs. - pub bloom_filter: BloomFilterConfig, + /// Bloom filter index configs. + pub bloom_filter_index: BloomFilterConfig, /// Memtable config pub memtable: MemtableConfig, @@ -157,7 +157,7 @@ impl Default for MitoConfig { index: IndexConfig::default(), inverted_index: InvertedIndexConfig::default(), fulltext_index: FulltextIndexConfig::default(), - bloom_filter: BloomFilterConfig::default(), + bloom_filter_index: BloomFilterConfig::default(), memtable: MemtableConfig::default(), min_compaction_interval: Duration::from_secs(0), }; diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index bae0047d1089..dd844a7d534c 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -360,7 +360,7 @@ impl RegionFlushTask { index_options: self.index_options.clone(), inverted_index_config: self.engine_config.inverted_index.clone(), fulltext_index_config: self.engine_config.fulltext_index.clone(), - bloom_filter_config: self.engine_config.bloom_filter.clone(), + bloom_filter_index_config: self.engine_config.bloom_filter_index.clone(), }; let Some(sst_info) = self .access_layer diff --git a/src/mito2/src/sst/index.rs b/src/mito2/src/sst/index.rs index daaba859f45e..e575ac687b8c 100644 --- a/src/mito2/src/sst/index.rs +++ b/src/mito2/src/sst/index.rs @@ -179,7 +179,7 @@ pub(crate) struct IndexerBuilder<'a> { pub(crate) index_options: IndexOptions, pub(crate) inverted_index_config: InvertedIndexConfig, pub(crate) fulltext_index_config: FulltextIndexConfig, - pub(crate) bloom_filter_config: BloomFilterConfig, + pub(crate) bloom_filter_index_config: BloomFilterConfig, } impl<'a> IndexerBuilder<'a> { @@ -322,8 +322,8 @@ impl<'a> IndexerBuilder<'a> { fn build_bloom_filter_indexer(&self) -> Option { let create = match self.op_type { - OperationType::Flush => self.bloom_filter_config.create_on_flush.auto(), - OperationType::Compact => self.bloom_filter_config.create_on_compaction.auto(), + OperationType::Flush => self.bloom_filter_index_config.create_on_flush.auto(), + OperationType::Compact => self.bloom_filter_index_config.create_on_compaction.auto(), }; if !create { @@ -334,7 +334,7 @@ impl<'a> IndexerBuilder<'a> { return None; } - let mem_limit = self.bloom_filter_config.mem_threshold_on_create(); + let mem_limit = self.bloom_filter_index_config.mem_threshold_on_create(); let indexer = BloomFilterIndexer::new( self.file_id, self.metadata, @@ -500,7 +500,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -535,7 +535,7 @@ mod tests { ..Default::default() }, fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -558,7 +558,7 @@ mod tests { create_on_compaction: Mode::Disable, ..Default::default() }, - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -578,7 +578,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig { + bloom_filter_index_config: BloomFilterConfig { create_on_compaction: Mode::Disable, ..Default::default() }, @@ -613,7 +613,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -638,7 +638,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -663,7 +663,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; @@ -695,7 +695,7 @@ mod tests { index_options: IndexOptions::default(), inverted_index_config: InvertedIndexConfig::default(), fulltext_index_config: FulltextIndexConfig::default(), - bloom_filter_config: BloomFilterConfig::default(), + bloom_filter_index_config: BloomFilterConfig::default(), } .build() .await; diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 611f73a3cb89..28593f13adb0 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -955,7 +955,7 @@ apply_on_query = "auto" mem_threshold_on_create = "auto" compress = true -[region_engine.mito.bloom_filter] +[region_engine.mito.bloom_filter_index] create_on_flush = "auto" create_on_compaction = "auto" apply_on_query = "auto"