diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index 19762c8f6bef4..d4ea2048d037e 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cell::RefCell; use std::sync::Arc; use itertools::Itertools; @@ -29,9 +28,9 @@ use crate::hummock::compaction::picker::TrivialMovePicker; use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig}; use crate::hummock::level_handler::LevelHandler; -std::thread_local! { - static LOG_COUNTER: RefCell = RefCell::new(0); -} +// std::thread_local! { +// static LOG_COUNTER: RefCell = RefCell::new(0); +// } pub struct LevelCompactionPicker { target_level: usize, @@ -258,17 +257,17 @@ impl LevelCompactionPicker { stats, ) { if l0.total_file_size > target_level.total_file_size * 4 { - let log_counter = LOG_COUNTER.with_borrow_mut(|counter| { - *counter += 1; - *counter - }); + // let log_counter = LOG_COUNTER.with_borrow_mut(|counter| { + // *counter += 1; + // *counter + // }); // reduce log - if log_counter % 2 == 0 && result.input_levels.len() > 0 { + if result.input_levels.len() > 0 { tracing::warn!("skip task with level count: {}, file count: {}, first level size: {}, select size: {}, target size: {}, target level size: {}", result.input_levels.len(), result.total_file_count, - result.input_levels[0].table_infos.iter().map(|sst|sst.file_size).sum::(), + l0.sub_levels[0].total_file_size, result.select_input_size, result.target_input_size, target_level.total_file_size, @@ -280,6 +279,19 @@ impl LevelCompactionPicker { return Some(result); } + if l0.total_file_size > target_level.total_file_size * 4 { + tracing::warn!( + "failed to select task. l0 sub levels info: {:?}", + l0.sub_levels + .iter() + .map(|level| ( + level.level_type(), + level.table_infos.len(), + level.total_file_size + )) + .collect_vec() + ); + } None } } diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index cae98e73afc52..c0b29f9680eca 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -45,14 +45,6 @@ impl CompactionPicker for IntraCompactionPicker { return None; } - let is_l0_pending_compact = - level_handlers[0].is_level_all_pending_compact(&l0.sub_levels[0]); - - if is_l0_pending_compact { - stats.skip_by_pending_files += 1; - return None; - } - if let Some(ret) = self.pick_l0_trivial_move_file(l0, level_handlers, stats) { return Some(ret); } @@ -237,7 +229,7 @@ impl IntraCompactionPicker { continue; } - if level_handlers[0].is_level_pending_compact(level) { + if level_handlers[0].is_level_all_pending_compact(level) { continue; } diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 994ca4297a502..25627d475b7b5 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::sync::Arc; use risingwave_hummock_sdk::append_sstable_info_to_string; @@ -381,6 +382,11 @@ impl NonOverlapSubLevelPicker { ret.total_file_count = total_file_count; ret.total_file_size = total_file_size; ret.sstable_infos.truncate(index + 1); + tracing::warn!( + "truncate files to meet: total_file_count: {}, total_file_size: {}", + total_file_count, + total_file_size + ); break; } } @@ -403,17 +409,27 @@ impl NonOverlapSubLevelPicker { } let mut scores = vec![]; + let mut select_files: HashSet = HashSet::default(); for sst in &l0[0].table_infos { if level_handler.is_pending_compact(&sst.sst_id) { continue; } + if select_files.contains(&sst.sst_id) { + continue; + } + let ret = self.pick_sub_level(l0, level_handler, sst); if ret.sstable_infos.len() < self.min_expected_level_count && ret.total_file_size < self.min_compaction_bytes { continue; } + + for sst in &ret.sstable_infos[0] { + select_files.insert(sst.sst_id); + } + scores.push(ret); }