chore(storage): Adjust the tier_compaction limit to reduce the impact on ci (#9534)

Author: Li0k

proto/hummock.proto

@@ -546,6 +546,7 @@ message RiseCtlUpdateCompactionConfigRequest {
       uint32 max_sub_compaction = 9;
       uint64 level0_stop_write_threshold_sub_level_number = 10;
       uint32 level0_sub_level_compact_level_count = 11;
+      uint32 level0_overlapping_sub_level_compact_level_count = 12;
     }
   }
   repeated uint64 compaction_group_ids = 1;
@@ -634,6 +635,9 @@ message CompactionConfig {
   uint64 level0_stop_write_threshold_sub_level_number = 15;
   uint64 level0_max_compact_file_number = 16;
   uint32 level0_sub_level_compact_level_count = 17;
+
+  // for tier compaction pick overlapping level
+  uint32 level0_overlapping_sub_level_compact_level_count = 18;
 }
 
 message TableStats {

src/meta/src/hummock/compaction/compaction_config.rs

@@ -30,6 +30,7 @@ const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * 1024 * 1024; // 512MB;
 const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 1000;
 const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 96;
 const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
+const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 6;
 
 pub struct CompactionConfigBuilder {
     config: CompactionConfig,
@@ -73,6 +74,8 @@ impl CompactionConfigBuilder {
                 // max_bytes_for_level_base
                 level0_max_compact_file_number: DEFAULT_MAX_COMPACTION_FILE_COUNT,
                 level0_sub_level_compact_level_count: DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT,
+                level0_overlapping_sub_level_compact_level_count:
+                    DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT,
             },
         }
     }
@@ -137,4 +140,5 @@ builder_field! {
     level0_stop_write_threshold_sub_level_number: u64,
     level0_max_compact_file_number: u64,
     level0_sub_level_compact_level_count: u32,
+    level0_overlapping_sub_level_compact_level_count: u32,
 }

src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs

@@ -115,9 +115,10 @@ impl LevelCompactionPicker {
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
         let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
+        // FIXME(li0k): Just workaround, need to use more reasonable way to limit task size
         let max_depth = std::cmp::max(
-            (self.config.max_compaction_bytes / self.config.sub_level_max_compaction_bytes)
-                as usize,
+            (self.config.max_compaction_bytes as f64
+                / self.config.sub_level_max_compaction_bytes as f64) as usize,
             self.config.level0_sub_level_compact_level_count as usize + 1,
         );
 

src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs

@@ -66,10 +66,14 @@ impl TierCompactionPicker {
                 self.config.max_compaction_bytes,
                 self.config.sub_level_max_compaction_bytes,
             );
+
+            let tier_sub_level_compact_level_count =
+                self.config.level0_sub_level_compact_level_count as usize;
+            // FIXME(li0k): Just workaround, need to use more reasonable way to limit task size
             let max_depth = std::cmp::max(
-                (self.config.max_compaction_bytes / self.config.sub_level_max_compaction_bytes)
-                    as usize,
-                self.config.level0_sub_level_compact_level_count as usize + 1,
+                (self.config.max_compaction_bytes as f64
+                    / self.config.sub_level_max_compaction_bytes as f64) as usize,
+                tier_sub_level_compact_level_count + 1,
             );
             let non_overlap_sub_level_picker = NonOverlapSubLevelPicker::new(
                 0,
@@ -89,6 +93,8 @@ impl TierCompactionPicker {
 
             let mut skip_by_write_amp = false;
             let mut level_select_tables: Vec<Vec<SstableInfo>> = vec![];
+            // Limit the number of selection levels for the non-overlapping
+            // sub_level at least level0_sub_level_compact_level_count
             for (plan_index, (compaction_bytes, all_file_count, level_select_sst)) in
                 l0_select_tables_vec.into_iter().enumerate()
             {
@@ -112,15 +118,14 @@ impl TierCompactionPicker {
 
                 // This limitation would keep our write-amplification no more than
                 // ln(max_compaction_bytes/flush_level_bytes) /
-                // ln(self.config.level0_tier_compact_file_number/2) Here we only use half
-                // of level0_tier_compact_file_number just for convenient.
+                // ln(self.config.level0_sub_level_compact_level_count/2) Here we only use half
+                // of level0_sub_level_compact_level_count just for convenient.
                 let is_write_amp_large =
                     max_level_size * self.config.level0_sub_level_compact_level_count as u64 / 2
                         >= compaction_bytes;
 
                 if is_write_amp_large
-                    && level_select_sst.len()
-                        < self.config.level0_sub_level_compact_level_count as usize
+                    && level_select_sst.len() < tier_sub_level_compact_level_count
                     && all_file_count < self.config.level0_max_compact_file_number as usize
                 {
                     skip_by_write_amp = true;
@@ -192,12 +197,6 @@ impl TierCompactionPicker {
                 continue;
             }
 
-            if select_tables.iter().map(|sst| sst.file_size).sum::<u64>()
-                < self.config.target_file_size_base
-            {
-                continue;
-            }
-
             // support trivial move cross multi sub_levels
             let mut overlap = self.overlap_strategy.create_overlap_info();
             for sst in &select_tables {
@@ -363,7 +362,7 @@ impl TierCompactionPicker {
             let max_compaction_bytes = std::cmp::min(
                 self.config.max_compaction_bytes,
                 self.config.sub_level_max_compaction_bytes
-                    * self.config.level0_sub_level_compact_level_count as u64,
+                    * self.config.level0_overlapping_sub_level_compact_level_count as u64,
             );
 
             let mut compaction_bytes = level.total_file_size;
@@ -407,7 +406,9 @@ impl TierCompactionPicker {
 
             // If waiting_enough_files is not satisfied, we will raise the priority of the number of
             // levels to ensure that we can merge as many sub_levels as possible
-            if select_level_inputs.len() < self.config.level0_sub_level_compact_level_count as usize
+            let tier_sub_level_compact_level_count =
+                self.config.level0_overlapping_sub_level_compact_level_count as usize;
+            if select_level_inputs.len() < tier_sub_level_compact_level_count
                 && waiting_enough_files
             {
                 stats.skip_by_count_limit += 1;
@@ -502,6 +503,7 @@ pub mod tests {
                 .level0_tier_compact_file_number(2)
                 .target_file_size_base(30)
                 .level0_sub_level_compact_level_count(2)
+                .level0_overlapping_sub_level_compact_level_count(4)
                 .build(),
         );
         let mut picker =
@@ -529,6 +531,8 @@ pub mod tests {
                 generate_table(2, 1, 150, 250, 1),
             ],
             vec![generate_table(3, 1, 10, 90, 1)],
+            vec![generate_table(4, 1, 10, 90, 1)],
+            vec![generate_table(5, 1, 10, 90, 1)],
         ]);
         let mut levels = Levels {
             l0: Some(l0),
@@ -577,6 +581,10 @@ pub mod tests {
                 generate_table(4, 1, 100, 200, 1),
                 generate_table(5, 1, 50, 150, 1),
             ],
+            vec![
+                generate_table(6, 1, 100, 200, 1),
+                generate_table(7, 1, 50, 150, 1),
+            ],
         ]);
         let levels = Levels {
             l0: Some(l0),
@@ -588,6 +596,7 @@ pub mod tests {
             CompactionConfigBuilder::new()
                 .level0_tier_compact_file_number(2)
                 .level0_sub_level_compact_level_count(2)
+                .level0_overlapping_sub_level_compact_level_count(4)
                 .build(),
         );
         let mut picker =
@@ -596,13 +605,13 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        assert_eq!(ret.input_levels.len(), 3);
+        assert_eq!(ret.input_levels.len(), 4);
         assert_eq!(
             ret.input_levels
                 .iter()
                 .map(|i| i.table_infos.len())
                 .sum::<usize>(),
-            5
+            7
         );
 
         let empty_level = Levels {
@@ -624,7 +633,10 @@ pub mod tests {
                     generate_table(1, 1, 100, 200, 1),
                     generate_table(2, 1, 250, 300, 1),
                 ],
-                vec![generate_table(3, 1, 10, 90, 1)],
+                vec![
+                    generate_table(3, 1, 10, 90, 1),
+                    generate_table(6, 1, 100, 110, 1),
+                ],
                 vec![
                     generate_table(4, 1, 50, 99, 1),
                     generate_table(5, 1, 100, 200, 1),
@@ -640,6 +652,7 @@ pub mod tests {
             let config = Arc::new(
                 CompactionConfigBuilder::new()
                     .level0_sub_level_compact_level_count(1)
+                    .level0_overlapping_sub_level_compact_level_count(4)
                     .build(),
             );
             let mut picker =
@@ -666,7 +679,10 @@ pub mod tests {
                     generate_table(1, 1, 100, 200, 1),
                     generate_table(2, 1, 300, 400, 1),
                 ],
-                vec![generate_table(3, 1, 100, 200, 1)],
+                vec![
+                    generate_table(3, 1, 100, 200, 1),
+                    generate_table(6, 1, 300, 500, 1),
+                ],
                 vec![
                     generate_table(4, 1, 100, 200, 1),
                     generate_table(5, 1, 300, 400, 1),
@@ -703,7 +719,7 @@ pub mod tests {
             assert_eq!(3, ret.input_levels[1].table_infos[0].get_sst_id());
             assert_eq!(1, ret.input_levels[2].table_infos[0].get_sst_id());
 
-            // will pick sst [2, 5]
+            // will pick sst [2, 6, 5]
             let ret2 = picker
                 .pick_compaction(&levels, &levels_handler, &mut local_stats)
                 .unwrap();
@@ -713,11 +729,12 @@ pub mod tests {
                     .iter()
                     .map(|i| i.table_infos.len())
                     .sum::<usize>(),
-                2
+                3
             );
 
             assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(2, ret2.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(6, ret2.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
         }
 
         {
@@ -798,6 +815,8 @@ pub mod tests {
             ],
             vec![generate_table(6, 1, 1, 100, 1)],
             vec![generate_table(7, 1, 1, 100, 1)],
+            vec![generate_table(8, 1, 1, 100, 1)],
+            vec![generate_table(9, 1, 1, 100, 1)],
         ]);
 
         let mut levels = Levels {
@@ -810,9 +829,10 @@ pub mod tests {
         let config = Arc::new(
             CompactionConfigBuilder::new()
                 .level0_tier_compact_file_number(2)
-                .sub_level_max_compaction_bytes(100)
+                .sub_level_max_compaction_bytes(500)
                 .max_compaction_bytes(500000)
                 .level0_sub_level_compact_level_count(2)
+                .level0_overlapping_sub_level_compact_level_count(4)
                 .build(),
         );
 
@@ -824,7 +844,7 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        assert_eq!(ret.input_levels.len(), 2);
+        assert_eq!(ret.input_levels.len(), 4);
         assert_eq!(ret.target_level, 0);
         assert_eq!(ret.target_sub_level_id, 1);
     }

src/meta/src/hummock/compaction_schedule_policy.rs

@@ -562,6 +562,7 @@ mod tests {
             .level0_tier_compact_file_number(1)
             .max_bytes_for_level_base(1)
             .level0_sub_level_compact_level_count(1)
+            .level0_overlapping_sub_level_compact_level_count(1)
             .build();
         let (_, hummock_manager, _, worker_node) = setup_compute_env_with_config(80, config).await;
         let context_id = worker_node.id;

src/meta/src/hummock/compaction_scheduler.rs

@@ -669,6 +669,7 @@ mod tests {
         );
 
         let _sst_infos = add_ssts(1, hummock_manager.as_ref(), context_id).await;
+
         let compactor = hummock_manager.get_idle_compactor().await.unwrap();
         // Cannot assign because of invalid compactor
         assert_matches!(

src/meta/src/hummock/manager/compaction_group_manager.rs

@@ -810,6 +810,9 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi
             MutableConfig::Level0SubLevelCompactLevelCount(c) => {
                 target.level0_sub_level_compact_level_count = *c;
             }
+            MutableConfig::Level0OverlappingSubLevelCompactLevelCount(c) => {
+                target.level0_overlapping_sub_level_compact_level_count = *c;
+            }
         }
     }
 }

src/meta/src/hummock/test_utils.rs

@@ -313,6 +313,7 @@ pub async fn setup_compute_env(
         .level0_tier_compact_file_number(1)
         .level0_max_compact_file_number(130)
         .level0_sub_level_compact_level_count(1)
+        .level0_overlapping_sub_level_compact_level_count(1)
         .build();
     setup_compute_env_with_config(port, config).await
 }