risingwave_meta/hummock/compaction/
compaction_config.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common::config::CompactionConfig as CompactionConfigOpt;
16use risingwave_common::config::meta::default::compaction_config;
17use risingwave_pb::hummock::CompactionConfig;
18use risingwave_pb::hummock::compaction_config::CompactionMode;
19
20pub struct CompactionConfigBuilder {
21    config: CompactionConfig,
22}
23
24impl CompactionConfigBuilder {
25    pub fn new() -> Self {
26        #[allow(deprecated)]
27        Self {
28            config: CompactionConfig {
29                max_bytes_for_level_base: compaction_config::max_bytes_for_level_base(),
30                max_bytes_for_level_multiplier: compaction_config::max_bytes_for_level_multiplier(),
31                max_level: compaction_config::max_level() as u64,
32                max_compaction_bytes: compaction_config::max_compaction_bytes(),
33                sub_level_max_compaction_bytes: compaction_config::sub_level_max_compaction_bytes(),
34                level0_tier_compact_file_number: compaction_config::level0_tier_compact_file_number(
35                ),
36                target_file_size_base: compaction_config::target_file_size_base(),
37                compaction_mode: CompactionMode::Range as i32,
38                // support compression setting per level
39                // L0/L1 and L2 do not use compression algorithms
40                // L3 - L4 use Lz4, else use Zstd
41                compression_algorithm: vec![
42                    "None".to_owned(),
43                    "None".to_owned(),
44                    "None".to_owned(),
45                    "Lz4".to_owned(),
46                    "Lz4".to_owned(),
47                    "Zstd".to_owned(),
48                    "Zstd".to_owned(),
49                ],
50                compaction_filter_mask: compaction_config::compaction_filter_mask(),
51                max_sub_compaction: compaction_config::max_sub_compaction(),
52                max_space_reclaim_bytes: compaction_config::max_space_reclaim_bytes(),
53                split_by_state_table: false,
54                split_weight_by_vnode: 0,
55                level0_stop_write_threshold_sub_level_number:
56                    compaction_config::level0_stop_write_threshold_sub_level_number(),
57                // This configure variable shall be larger than level0_tier_compact_file_number, and
58                // it shall meet the following condition:
59                //    level0_max_compact_file_number * target_file_size_base >
60                // max_bytes_for_level_base
61                level0_max_compact_file_number: compaction_config::level0_max_compact_file_number(),
62                level0_sub_level_compact_level_count:
63                    compaction_config::level0_sub_level_compact_level_count(),
64                level0_overlapping_sub_level_compact_level_count:
65                    compaction_config::level0_overlapping_sub_level_compact_level_count(),
66                tombstone_reclaim_ratio: compaction_config::tombstone_reclaim_ratio(),
67                enable_emergency_picker: compaction_config::enable_emergency_picker(),
68                max_l0_compact_level_count: Some(compaction_config::max_l0_compact_level_count()),
69                sst_allowed_trivial_move_min_size: Some(
70                    compaction_config::sst_allowed_trivial_move_min_size(),
71                ),
72                disable_auto_group_scheduling: Some(
73                    compaction_config::disable_auto_group_scheduling(),
74                ),
75                max_overlapping_level_size: Some(compaction_config::max_overlapping_level_size()),
76                sst_allowed_trivial_move_max_count: Some(
77                    compaction_config::sst_allowed_trivial_move_max_count(),
78                ),
79                emergency_level0_sst_file_count: Some(
80                    compaction_config::emergency_level0_sst_file_count(),
81                ),
82                emergency_level0_sub_level_partition: Some(
83                    compaction_config::emergency_level0_sub_level_partition(),
84                ),
85                level0_stop_write_threshold_max_sst_count: Some(
86                    compaction_config::level0_stop_write_threshold_max_sst_count(),
87                ),
88                level0_stop_write_threshold_max_size: Some(
89                    compaction_config::level0_stop_write_threshold_max_size(),
90                ),
91                enable_optimize_l0_interval_selection: Some(
92                    compaction_config::enable_optimize_l0_interval_selection(),
93                ),
94                vnode_aligned_level_size_threshold: None,
95                max_kv_count_for_xor16: compaction_config::max_kv_count_for_xor16(),
96                max_vnode_key_range_bytes: compaction_config::max_vnode_key_range_bytes(),
97            },
98        }
99    }
100
101    pub fn with_config(config: CompactionConfig) -> Self {
102        Self { config }
103    }
104
105    pub fn with_opt(opt: &CompactionConfigOpt) -> Self {
106        Self::new()
107            .max_bytes_for_level_base(opt.max_bytes_for_level_base)
108            .max_bytes_for_level_multiplier(opt.max_bytes_for_level_multiplier)
109            .max_compaction_bytes(opt.max_compaction_bytes)
110            .sub_level_max_compaction_bytes(opt.sub_level_max_compaction_bytes)
111            .level0_tier_compact_file_number(opt.level0_tier_compact_file_number)
112            .target_file_size_base(opt.target_file_size_base)
113            .compaction_filter_mask(opt.compaction_filter_mask)
114            .max_sub_compaction(opt.max_sub_compaction)
115            .level0_stop_write_threshold_sub_level_number(
116                opt.level0_stop_write_threshold_sub_level_number,
117            )
118            .level0_sub_level_compact_level_count(opt.level0_sub_level_compact_level_count)
119            .level0_overlapping_sub_level_compact_level_count(
120                opt.level0_overlapping_sub_level_compact_level_count,
121            )
122            .max_space_reclaim_bytes(opt.max_space_reclaim_bytes)
123            .level0_max_compact_file_number(opt.level0_max_compact_file_number)
124            .tombstone_reclaim_ratio(opt.tombstone_reclaim_ratio)
125            .max_level(opt.max_level as u64)
126            .sst_allowed_trivial_move_min_size(Some(opt.sst_allowed_trivial_move_min_size))
127            .sst_allowed_trivial_move_max_count(Some(opt.sst_allowed_trivial_move_max_count))
128            .max_overlapping_level_size(Some(opt.max_overlapping_level_size))
129            .emergency_level0_sst_file_count(Some(opt.emergency_level0_sst_file_count))
130            .emergency_level0_sub_level_partition(Some(opt.emergency_level0_sub_level_partition))
131            .level0_stop_write_threshold_max_sst_count(Some(
132                opt.level0_stop_write_threshold_max_sst_count,
133            ))
134            .level0_stop_write_threshold_max_size(Some(opt.level0_stop_write_threshold_max_size))
135            .enable_optimize_l0_interval_selection(Some(opt.enable_optimize_l0_interval_selection))
136            .max_kv_count_for_xor16(opt.max_kv_count_for_xor16)
137            .max_vnode_key_range_bytes(opt.max_vnode_key_range_bytes)
138    }
139
140    pub fn build(self) -> CompactionConfig {
141        if let Err(reason) = validate_compaction_config(&self.config) {
142            tracing::warn!("Bad compaction config: {}", reason);
143        }
144        self.config
145    }
146}
147
148/// Returns Ok if `config` is valid,
149/// or the reason why it's invalid.
150pub fn validate_compaction_config(config: &CompactionConfig) -> Result<(), String> {
151    let sub_level_number_threshold_min = 1;
152    if config.level0_stop_write_threshold_sub_level_number < sub_level_number_threshold_min {
153        return Err(format!(
154            "{} is too small for level0_stop_write_threshold_sub_level_number, expect >= {}",
155            config.level0_stop_write_threshold_sub_level_number, sub_level_number_threshold_min
156        ));
157    }
158    Ok(())
159}
160
161impl Default for CompactionConfigBuilder {
162    fn default() -> Self {
163        Self::new()
164    }
165}
166
167macro_rules! builder_field {
168    ($( $name:ident: $type:ty ),* ,) => {
169        impl CompactionConfigBuilder {
170            $(
171                pub fn $name(mut self, v:$type) -> Self {
172                    self.config.$name = v;
173                    self
174                }
175            )*
176        }
177    }
178}
179
180builder_field! {
181    max_bytes_for_level_base: u64,
182    max_bytes_for_level_multiplier: u64,
183    max_level: u64,
184    max_compaction_bytes: u64,
185    sub_level_max_compaction_bytes: u64,
186    level0_tier_compact_file_number: u64,
187    compaction_mode: i32,
188    compression_algorithm: Vec<String>,
189    compaction_filter_mask: u32,
190    target_file_size_base: u64,
191    max_sub_compaction: u32,
192    max_space_reclaim_bytes: u64,
193    level0_stop_write_threshold_sub_level_number: u64,
194    level0_max_compact_file_number: u64,
195    level0_sub_level_compact_level_count: u32,
196    level0_overlapping_sub_level_compact_level_count: u32,
197    tombstone_reclaim_ratio: u32,
198    sst_allowed_trivial_move_min_size: Option<u64>,
199    sst_allowed_trivial_move_max_count: Option<u32>,
200    disable_auto_group_scheduling: Option<bool>,
201    max_overlapping_level_size: Option<u64>,
202    emergency_level0_sst_file_count: Option<u32>,
203    emergency_level0_sub_level_partition: Option<u32>,
204    level0_stop_write_threshold_max_sst_count: Option<u32>,
205    level0_stop_write_threshold_max_size: Option<u64>,
206    enable_optimize_l0_interval_selection: Option<bool>,
207    max_kv_count_for_xor16: Option<u64>,
208    max_vnode_key_range_bytes: Option<u64>,
209}
risingwave_meta/hummock/compaction/compaction_config.rs

risingwave_meta/hummock/compaction/
compaction_config.rs
