test small tier number

Signed-off-by: Little-Wallace <[email protected]>

src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::cell::RefCell;
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -28,9 +29,9 @@ use crate::hummock::compaction::picker::TrivialMovePicker;
 use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig};
 use crate::hummock::level_handler::LevelHandler;
 
-// std::thread_local! {
-//     static LOG_COUNTER: RefCell<usize> = RefCell::new(0);
-// }
+std::thread_local! {
+    static LOG_COUNTER: RefCell<usize> = RefCell::new(0);
+}
 
 pub struct LevelCompactionPicker {
     target_level: usize,
@@ -256,18 +257,17 @@ impl LevelCompactionPicker {
                 ValidationRuleType::ToBase,
                 stats,
             ) {
-                if l0.total_file_size > target_level.total_file_size * 4 {
-                    // let log_counter = LOG_COUNTER.with_borrow_mut(|counter| {
-                    //     *counter += 1;
-                    //     *counter
-                    // });
+                if l0.total_file_size > target_level.total_file_size * 8 {
+                    let log_counter = LOG_COUNTER.with_borrow_mut(|counter| {
+                        *counter += 1;
+                        *counter
+                    });
 
                     // reduce log
-                    if result.input_levels.len() > 0 {
-                        tracing::warn!("skip task with level count: {}, file count: {}, first level size: {}, select size: {}, target size: {}, target level size: {}",
+                    if log_counter % 100 == 0 {
+                        tracing::warn!("skip task with level count: {}, file count: {}, select size: {}, target size: {}, target level size: {}",
                             result.input_levels.len(),
                             result.total_file_count,
-                            l0.sub_levels[0].total_file_size,
                             result.select_input_size,
                             result.target_input_size,
                             target_level.total_file_size,
@@ -279,19 +279,6 @@ impl LevelCompactionPicker {
 
             return Some(result);
         }
-        if l0.total_file_size > target_level.total_file_size * 4 {
-            tracing::warn!(
-                "failed to select task. l0 sub levels info: {:?}",
-                l0.sub_levels
-                    .iter()
-                    .map(|level| (
-                        level.level_type(),
-                        level.table_infos.len(),
-                        level.total_file_size
-                    ))
-                    .collect_vec()
-            );
-        }
         None
     }
 }

src/meta/src/hummock/compaction/picker/compaction_task_validator.rs

@@ -101,19 +101,14 @@ impl CompactionTaskValidationRule for TierCompactionTaskValidationRule {
             return true;
         }
 
-        // so the design here wants to merge multiple overlapping-levels in one compaction
-        let max_compaction_bytes = std::cmp::min(
-            self.config.max_compaction_bytes,
-            self.config.sub_level_max_compaction_bytes
-                * self.config.level0_overlapping_sub_level_compact_level_count as u64,
-        );
-
         // If waiting_enough_files is not satisfied, we will raise the priority of the number of
         // levels to ensure that we can merge as many sub_levels as possible
         let tier_sub_level_compact_level_count =
             self.config.level0_overlapping_sub_level_compact_level_count as usize;
-        if input.input_levels.len() < tier_sub_level_compact_level_count
-            && input.select_input_size < max_compaction_bytes
+        let min_sub_level_compact_level_count = tier_sub_level_compact_level_count / 2;
+        if input.input_levels.len() < min_sub_level_compact_level_count
+            || (input.input_levels.len() < tier_sub_level_compact_level_count
+                && input.select_input_size < self.config.max_bytes_for_level_base)
         {
             stats.skip_by_count_limit += 1;
             return false;

src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use risingwave_hummock_sdk::append_sstable_info_to_string;
@@ -382,11 +381,6 @@ impl NonOverlapSubLevelPicker {
                     ret.total_file_count = total_file_count;
                     ret.total_file_size = total_file_size;
                     ret.sstable_infos.truncate(index + 1);
-                    tracing::warn!(
-                        "truncate files to meet: total_file_count: {}, total_file_size: {}",
-                        total_file_count,
-                        total_file_size
-                    );
                     break;
                 }
             }
@@ -409,27 +403,17 @@ impl NonOverlapSubLevelPicker {
         }
 
         let mut scores = vec![];
-        let mut select_files: HashSet<u64> = HashSet::default();
         for sst in &l0[0].table_infos {
             if level_handler.is_pending_compact(&sst.sst_id) {
                 continue;
             }
 
-            if select_files.contains(&sst.sst_id) {
-                continue;
-            }
-
             let ret = self.pick_sub_level(l0, level_handler, sst);
             if ret.sstable_infos.len() < self.min_expected_level_count
                 && ret.total_file_size < self.min_compaction_bytes
             {
                 continue;
             }
-
-            for sst in &ret.sstable_infos[0] {
-                select_files.insert(sst.sst_id);
-            }
-
             scores.push(ret);
         }