risingwave_meta/hummock/compaction/picker/
trivial_move_compaction_picker.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16
17use risingwave_hummock_sdk::level::InputLevel;
18use risingwave_hummock_sdk::sstable_info::SstableInfo;
19use risingwave_pb::hummock::LevelType;
20
21use super::{CompactionInput, LocalPickerStatistic};
22use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
23use crate::hummock::level_handler::LevelHandler;
24
25pub struct TrivialMovePicker {
26    level: usize,
27    target_level: usize,
28    overlap_strategy: Arc<dyn OverlapStrategy>,
29
30    // The minimum size of sst that can be selected as trivial move.
31    sst_allowed_trivial_move_min_size: u64,
32
33    sst_allowed_trivial_move_max_count: usize,
34}
35
36impl TrivialMovePicker {
37    pub fn new(
38        level: usize,
39        target_level: usize,
40        overlap_strategy: Arc<dyn OverlapStrategy>,
41        sst_allowed_trivial_move_min_size: u64,
42        sst_allowed_trivial_move_max_count: usize,
43    ) -> Self {
44        Self {
45            level,
46            target_level,
47            overlap_strategy,
48            sst_allowed_trivial_move_min_size,
49            sst_allowed_trivial_move_max_count,
50        }
51    }
52
53    pub fn pick_multi_trivial_move_ssts(
54        &self,
55        select_tables: &[SstableInfo],
56        target_tables: &[SstableInfo],
57        level_handlers: &[LevelHandler],
58        stats: &mut LocalPickerStatistic,
59    ) -> Option<Vec<SstableInfo>> {
60        let mut skip_by_pending = false;
61
62        // means we have already found some sst but not match `sst_allowed_trivial_move_max_count`.
63        let mut skip_by_size = false;
64        let mut result = vec![];
65
66        let mut overlap_info = self.overlap_strategy.create_overlap_info();
67        for sst in select_tables {
68            if result.len() >= self.sst_allowed_trivial_move_max_count {
69                break;
70            }
71
72            // find the first sst that can be trivial moved.
73            if sst.sst_size < self.sst_allowed_trivial_move_min_size {
74                skip_by_size = true;
75
76                // Stop probing if we have already found some sst to move. And should avoid small sst move to target level.
77                if !result.is_empty() {
78                    break;
79                }
80
81                continue;
82            }
83
84            if level_handlers[self.level].is_pending_compact(&sst.sst_id) {
85                skip_by_pending = true;
86
87                if !result.is_empty() {
88                    break;
89                }
90
91                continue;
92            }
93            overlap_info.update(sst);
94            let overlap_files_range = overlap_info.check_multiple_overlap(target_tables);
95
96            if overlap_files_range.is_empty() {
97                result.push(sst.clone());
98            } else {
99                // stop probing if we have already found some sst to move.
100                if !result.is_empty() {
101                    break;
102                }
103
104                // reset overlap_info
105                overlap_info = self.overlap_strategy.create_overlap_info();
106            }
107        }
108
109        if !result.is_empty() {
110            return Some(result);
111        }
112
113        if skip_by_pending {
114            stats.skip_by_pending_files += 1;
115        }
116
117        if skip_by_size {
118            stats.skip_by_count_limit += 1;
119        }
120
121        None
122    }
123
124    pub fn pick_trivial_move_task(
125        &self,
126        select_tables: &[SstableInfo],
127        target_tables: &[SstableInfo],
128        level_handlers: &[LevelHandler],
129        stats: &mut LocalPickerStatistic,
130    ) -> Option<CompactionInput> {
131        if let Some(trivial_move_ssts) =
132            self.pick_multi_trivial_move_ssts(select_tables, target_tables, level_handlers, stats)
133        {
134            return Some(CompactionInput {
135                select_input_size: trivial_move_ssts.iter().map(|s| s.sst_size).sum(),
136                total_file_count: 1,
137                input_levels: vec![
138                    InputLevel {
139                        level_idx: self.level as u32,
140                        level_type: LevelType::Nonoverlapping,
141                        table_infos: trivial_move_ssts,
142                    },
143                    InputLevel {
144                        level_idx: self.target_level as u32,
145                        level_type: LevelType::Nonoverlapping,
146                        table_infos: vec![],
147                    },
148                ],
149                target_level: self.target_level,
150                ..Default::default()
151            });
152        }
153
154        None
155    }
156}
157
158#[cfg(test)]
159pub mod tests {
160    use std::sync::Arc;
161
162    use risingwave_hummock_sdk::sstable_info::{SstableInfo, SstableInfoInner};
163
164    use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
165    use crate::hummock::compaction::create_overlap_strategy;
166    use crate::hummock::level_handler::LevelHandler;
167
168    #[test]
169    fn test_allowed_trivial_move_min_size() {
170        let sst: SstableInfo = SstableInfoInner {
171            sst_id: 1,
172            file_size: 100,
173            sst_size: 100,
174            ..Default::default()
175        }
176        .into();
177
178        let config = Arc::new(
179            CompactionConfigBuilder::new()
180                .level0_tier_compact_file_number(2)
181                .level0_sub_level_compact_level_count(1)
182                .build(),
183        );
184        let levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
185        let overlap_strategy = create_overlap_strategy(config.compaction_mode());
186        {
187            let trivial_move_picker =
188                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 200, 1);
189            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
190                &[sst.clone()],
191                &[],
192                &levels_handler,
193                &mut Default::default(),
194            );
195
196            assert!(trivial_move_task.is_none());
197        }
198
199        {
200            let trivial_move_picker =
201                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 50, 1);
202
203            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
204                &[sst.clone()],
205                &[],
206                &levels_handler,
207                &mut Default::default(),
208            );
209
210            assert!(trivial_move_task.is_some());
211        }
212    }
213
214    #[test]
215    fn test_pick_multi_trivial_move_sst() {
216        let sst1: SstableInfo = SstableInfoInner {
217            sst_id: 1,
218            file_size: 100,
219            sst_size: 100,
220            ..Default::default()
221        }
222        .into();
223        let sst2: SstableInfo = SstableInfoInner {
224            sst_id: 2,
225            file_size: 100,
226            sst_size: 100,
227            ..Default::default()
228        }
229        .into();
230        let sst3: SstableInfo = SstableInfoInner {
231            sst_id: 3,
232            file_size: 100,
233            sst_size: 100,
234            ..Default::default()
235        }
236        .into();
237        let sst4: SstableInfo = SstableInfoInner {
238            sst_id: 4,
239            file_size: 100,
240            sst_size: 100,
241            ..Default::default()
242        }
243        .into();
244
245        let config = Arc::new(
246            CompactionConfigBuilder::new()
247                .level0_tier_compact_file_number(2)
248                .level0_sub_level_compact_level_count(1)
249                .sst_allowed_trivial_move_min_size(Some(50))
250                .build(),
251        );
252        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
253        let overlap_strategy = create_overlap_strategy(config.compaction_mode());
254        {
255            let trivial_move_picker =
256                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 200, 10);
257            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
258                &[sst1.clone(), sst2.clone(), sst3.clone(), sst4.clone()],
259                &[],
260                &levels_handler,
261                &mut Default::default(),
262            );
263
264            assert!(trivial_move_task.is_none());
265        }
266
267        {
268            let trivial_move_picker =
269                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 50, 10);
270
271            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
272                &[sst1.clone(), sst2.clone(), sst3.clone(), sst4.clone()],
273                &[],
274                &levels_handler,
275                &mut Default::default(),
276            );
277
278            assert!(trivial_move_task.is_some());
279            assert_eq!(trivial_move_task.unwrap().len(), 4);
280        }
281
282        {
283            let trivial_move_picker =
284                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 50, 2);
285
286            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
287                &[sst1.clone(), sst2.clone(), sst3.clone(), sst4.clone()],
288                &[],
289                &levels_handler,
290                &mut Default::default(),
291            );
292
293            assert!(trivial_move_task.is_some());
294            assert_eq!(trivial_move_task.unwrap().len(), 2);
295        }
296
297        {
298            levels_handler[0].test_add_pending_sst(2, 1);
299            let trivial_move_picker =
300                super::TrivialMovePicker::new(0, 1, overlap_strategy.clone(), 50, 4);
301            let trivial_move_task = trivial_move_picker.pick_multi_trivial_move_ssts(
302                &[sst1.clone(), sst2.clone(), sst3.clone(), sst4.clone()],
303                &[],
304                &levels_handler,
305                &mut Default::default(),
306            );
307
308            assert!(trivial_move_task.is_some());
309            assert_eq!(trivial_move_task.unwrap().len(), 1);
310        }
311    }
312}
risingwave_meta/hummock/compaction/picker/trivial_move_compaction_picker.rs

risingwave_meta/hummock/compaction/picker/
trivial_move_compaction_picker.rs
