fix pending level trivial move

Signed-off-by: Little-Wallace <[email protected]>
risingwavelabs · May 8, 2024 · c6374cf · c6374cf
1 parent 8babda9
commit c6374cf
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 19 deletions.
diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cell::RefCell;
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -29,9 +28,9 @@ use crate::hummock::compaction::picker::TrivialMovePicker;
 use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig};
 use crate::hummock::level_handler::LevelHandler;
 
-std::thread_local! {
-    static LOG_COUNTER: RefCell<usize> = RefCell::new(0);
-}
+// std::thread_local! {
+//     static LOG_COUNTER: RefCell<usize> = RefCell::new(0);
+// }
 
 pub struct LevelCompactionPicker {
     target_level: usize,
@@ -258,17 +257,17 @@ impl LevelCompactionPicker {
                 stats,
             ) {
                 if l0.total_file_size > target_level.total_file_size * 4 {
-                    let log_counter = LOG_COUNTER.with_borrow_mut(|counter| {
-                        *counter += 1;
-                        *counter
-                    });
+                    // let log_counter = LOG_COUNTER.with_borrow_mut(|counter| {
+                    //     *counter += 1;
+                    //     *counter
+                    // });
 
                     // reduce log
-                    if log_counter % 2 == 0 && result.input_levels.len() > 0 {
+                    if result.input_levels.len() > 0 {
                         tracing::warn!("skip task with level count: {}, file count: {}, first level size: {}, select size: {}, target size: {}, target level size: {}",
                             result.input_levels.len(),
                             result.total_file_count,
-                            result.input_levels[0].table_infos.iter().map(|sst|sst.file_size).sum::<u64>(),
+                            l0.sub_levels[0].total_file_size,
                             result.select_input_size,
                             result.target_input_size,
                             target_level.total_file_size,
@@ -280,6 +279,19 @@ impl LevelCompactionPicker {
 
             return Some(result);
         }
+        if l0.total_file_size > target_level.total_file_size * 4 {
+            tracing::warn!(
+                "failed to select task. l0 sub levels info: {:?}",
+                l0.sub_levels
+                    .iter()
+                    .map(|level| (
+                        level.level_type(),
+                        level.table_infos.len(),
+                        level.total_file_size
+                    ))
+                    .collect_vec()
+            );
+        }
         None
     }
 }

diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
@@ -45,14 +45,6 @@ impl CompactionPicker for IntraCompactionPicker {
             return None;
         }
 
-        let is_l0_pending_compact =
-            level_handlers[0].is_level_all_pending_compact(&l0.sub_levels[0]);
-
-        if is_l0_pending_compact {
-            stats.skip_by_pending_files += 1;
-            return None;
-        }
-
         if let Some(ret) = self.pick_l0_trivial_move_file(l0, level_handlers, stats) {
             return Some(ret);
         }
@@ -237,7 +229,7 @@ impl IntraCompactionPicker {
                 continue;
             }
 
-            if level_handlers[0].is_level_pending_compact(level) {
+            if level_handlers[0].is_level_all_pending_compact(level) {
                 continue;
             }
 

diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use risingwave_hummock_sdk::append_sstable_info_to_string;
@@ -381,6 +382,11 @@ impl NonOverlapSubLevelPicker {
                     ret.total_file_count = total_file_count;
                     ret.total_file_size = total_file_size;
                     ret.sstable_infos.truncate(index + 1);
+                    tracing::warn!(
+                        "truncate files to meet: total_file_count: {}, total_file_size: {}",
+                        total_file_count,
+                        total_file_size
+                    );
                     break;
                 }
             }
@@ -403,17 +409,27 @@ impl NonOverlapSubLevelPicker {
         }
 
         let mut scores = vec![];
+        let mut select_files: HashSet<u64> = HashSet::default();
         for sst in &l0[0].table_infos {
             if level_handler.is_pending_compact(&sst.sst_id) {
                 continue;
             }
 
+            if select_files.contains(&sst.sst_id) {
+                continue;
+            }
+
             let ret = self.pick_sub_level(l0, level_handler, sst);
             if ret.sstable_infos.len() < self.min_expected_level_count
                 && ret.total_file_size < self.min_compaction_bytes
             {
                 continue;
             }
+
+            for sst in &ret.sstable_infos[0] {
+                select_files.insert(sst.sst_id);
+            }
+
             scores.push(ret);
         }