Skip to content

Commit

Permalink
address comments
Browse files Browse the repository at this point in the history
Signed-off-by: Zhenchi <[email protected]>
  • Loading branch information
zhongzc committed Dec 26, 2024
1 parent 664975c commit e178ae9
Show file tree
Hide file tree
Showing 9 changed files with 35 additions and 34 deletions.
18 changes: 9 additions & 9 deletions config/datanode.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -576,37 +576,37 @@ apply_on_query = "auto"
## - `[size]` e.g. `64MB`: fixed memory threshold
mem_threshold_on_create = "auto"

## The options for bloom filter in Mito engine.
[region_engine.mito.bloom_filter]
## The options for bloom filter index in Mito engine.
[region_engine.mito.bloom_filter_index]

## Whether to create the bloom filter on flush.
## Whether to create the index on flush.
## - `auto`: automatically (default)
## - `disable`: never
create_on_flush = "auto"

## Whether to create the bloom filter on compaction.
## Whether to create the index on compaction.
## - `auto`: automatically (default)
## - `disable`: never
create_on_compaction = "auto"

## Whether to apply the bloom filter on query
## Whether to apply the index on query
## - `auto`: automatically (default)
## - `disable`: never
apply_on_query = "auto"

## Memory threshold for bloom filter creation.
## Memory threshold for the index creation.
## - `auto`: automatically determine the threshold based on the system memory size (default)
## - `unlimited`: no memory limit
## - `[size]` e.g. `64MB`: fixed memory threshold
mem_threshold_on_create = "auto"

## Cache size for bloom filter metadata.
## Cache size for the index metadata.
metadata_cache_size = "4MiB"

## Cache size for bloom filter content.
## Cache size for the index content.
content_cache_size = "128MiB"

## Page size for bloom filter content cache.
## Page size for the index content cache.
content_cache_page_size = "8MiB"

[region_engine.mito.memtable]
Expand Down
2 changes: 1 addition & 1 deletion config/standalone.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -620,7 +620,7 @@ apply_on_query = "auto"
mem_threshold_on_create = "auto"

## The options for bloom filter in Mito engine.
[region_engine.mito.bloom_filter]
[region_engine.mito.bloom_filter_index]

## Whether to create the bloom filter on flush.
## - `auto`: automatically (default)
Expand Down
4 changes: 2 additions & 2 deletions src/mito2/src/access_layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ impl AccessLayer {
index_options: request.index_options,
inverted_index_config: request.inverted_index_config,
fulltext_index_config: request.fulltext_index_config,
bloom_filter_config: request.bloom_filter_config,
bloom_filter_index_config: request.bloom_filter_index_config,
}
.build()
.await;
Expand Down Expand Up @@ -199,7 +199,7 @@ pub(crate) struct SstWriteRequest {
pub(crate) index_options: IndexOptions,
pub(crate) inverted_index_config: InvertedIndexConfig,
pub(crate) fulltext_index_config: FulltextIndexConfig,
pub(crate) bloom_filter_config: BloomFilterConfig,
pub(crate) bloom_filter_index_config: BloomFilterConfig,
}

pub(crate) async fn new_fs_cache_store(root: &str) -> Result<ObjectStore> {
Expand Down
6 changes: 3 additions & 3 deletions src/mito2/src/cache/write_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ impl WriteCache {
index_options: write_request.index_options,
inverted_index_config: write_request.inverted_index_config,
fulltext_index_config: write_request.fulltext_index_config,
bloom_filter_config: write_request.bloom_filter_config,
bloom_filter_index_config: write_request.bloom_filter_index_config,
}
.build()
.await;
Expand Down Expand Up @@ -379,7 +379,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: Default::default(),
fulltext_index_config: Default::default(),
bloom_filter_config: Default::default(),
bloom_filter_index_config: Default::default(),
};

let upload_request = SstUploadRequest {
Expand Down Expand Up @@ -472,7 +472,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: Default::default(),
fulltext_index_config: Default::default(),
bloom_filter_config: Default::default(),
bloom_filter_index_config: Default::default(),
};
let write_opts = WriteOptions {
row_group_size: 512,
Expand Down
5 changes: 3 additions & 2 deletions src/mito2/src/compaction/compactor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,8 @@ impl Compactor for DefaultCompactor {
let merge_mode = compaction_region.current_version.options.merge_mode();
let inverted_index_config = compaction_region.engine_config.inverted_index.clone();
let fulltext_index_config = compaction_region.engine_config.fulltext_index.clone();
let bloom_filter_config = compaction_region.engine_config.bloom_filter.clone();
let bloom_filter_index_config =
compaction_region.engine_config.bloom_filter_index.clone();
futs.push(async move {
let reader = CompactionSstReaderBuilder {
metadata: region_metadata.clone(),
Expand All @@ -326,7 +327,7 @@ impl Compactor for DefaultCompactor {
index_options,
inverted_index_config,
fulltext_index_config,
bloom_filter_config,
bloom_filter_index_config,
},
&write_opts,
)
Expand Down
6 changes: 3 additions & 3 deletions src/mito2/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ pub struct MitoConfig {
pub inverted_index: InvertedIndexConfig,
/// Full-text index configs.
pub fulltext_index: FulltextIndexConfig,
/// Bloom filter configs.
pub bloom_filter: BloomFilterConfig,
/// Bloom filter index configs.
pub bloom_filter_index: BloomFilterConfig,

/// Memtable config
pub memtable: MemtableConfig,
Expand Down Expand Up @@ -157,7 +157,7 @@ impl Default for MitoConfig {
index: IndexConfig::default(),
inverted_index: InvertedIndexConfig::default(),
fulltext_index: FulltextIndexConfig::default(),
bloom_filter: BloomFilterConfig::default(),
bloom_filter_index: BloomFilterConfig::default(),
memtable: MemtableConfig::default(),
min_compaction_interval: Duration::from_secs(0),
};
Expand Down
2 changes: 1 addition & 1 deletion src/mito2/src/flush.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ impl RegionFlushTask {
index_options: self.index_options.clone(),
inverted_index_config: self.engine_config.inverted_index.clone(),
fulltext_index_config: self.engine_config.fulltext_index.clone(),
bloom_filter_config: self.engine_config.bloom_filter.clone(),
bloom_filter_index_config: self.engine_config.bloom_filter_index.clone(),
};
let Some(sst_info) = self
.access_layer
Expand Down
24 changes: 12 additions & 12 deletions src/mito2/src/sst/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ pub(crate) struct IndexerBuilder<'a> {
pub(crate) index_options: IndexOptions,
pub(crate) inverted_index_config: InvertedIndexConfig,
pub(crate) fulltext_index_config: FulltextIndexConfig,
pub(crate) bloom_filter_config: BloomFilterConfig,
pub(crate) bloom_filter_index_config: BloomFilterConfig,
}

impl<'a> IndexerBuilder<'a> {
Expand Down Expand Up @@ -322,8 +322,8 @@ impl<'a> IndexerBuilder<'a> {

fn build_bloom_filter_indexer(&self) -> Option<BloomFilterIndexer> {
let create = match self.op_type {
OperationType::Flush => self.bloom_filter_config.create_on_flush.auto(),
OperationType::Compact => self.bloom_filter_config.create_on_compaction.auto(),
OperationType::Flush => self.bloom_filter_index_config.create_on_flush.auto(),
OperationType::Compact => self.bloom_filter_index_config.create_on_compaction.auto(),
};

if !create {
Expand All @@ -334,7 +334,7 @@ impl<'a> IndexerBuilder<'a> {
return None;
}

let mem_limit = self.bloom_filter_config.mem_threshold_on_create();
let mem_limit = self.bloom_filter_index_config.mem_threshold_on_create();
let indexer = BloomFilterIndexer::new(
self.file_id,
self.metadata,
Expand Down Expand Up @@ -500,7 +500,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand Down Expand Up @@ -535,7 +535,7 @@ mod tests {
..Default::default()
},
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand All @@ -558,7 +558,7 @@ mod tests {
create_on_compaction: Mode::Disable,
..Default::default()
},
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand All @@ -578,7 +578,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig {
bloom_filter_index_config: BloomFilterConfig {
create_on_compaction: Mode::Disable,
..Default::default()
},
Expand Down Expand Up @@ -613,7 +613,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand All @@ -638,7 +638,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand All @@ -663,7 +663,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand Down Expand Up @@ -695,7 +695,7 @@ mod tests {
index_options: IndexOptions::default(),
inverted_index_config: InvertedIndexConfig::default(),
fulltext_index_config: FulltextIndexConfig::default(),
bloom_filter_config: BloomFilterConfig::default(),
bloom_filter_index_config: BloomFilterConfig::default(),
}
.build()
.await;
Expand Down
2 changes: 1 addition & 1 deletion tests-integration/tests/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,7 @@ apply_on_query = "auto"
mem_threshold_on_create = "auto"
compress = true
[region_engine.mito.bloom_filter]
[region_engine.mito.bloom_filter_index]
create_on_flush = "auto"
create_on_compaction = "auto"
apply_on_query = "auto"
Expand Down

0 comments on commit e178ae9

Please sign in to comment.