risingwave_hummock_sdk/
filter_utils.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common::config::meta::default::compaction_config;
16use risingwave_pb::hummock::{CompactionConfig, PbSstableFilterLayout, PbSstableFilterType};
17
18/// Determines whether the key count is large enough to warrant using a blocked xor filter.
19///
20/// # Arguments
21/// * `kv_count` - The total number of keys
22/// * `blocked_kv_count_threshold` - Optional configured threshold. If None, uses
23///   `DEFAULT_BLOCKED_XOR_FILTER_KV_COUNT_THRESHOLD`.
24///
25/// # Returns
26/// `true` if `kv_count` exceeds the threshold, indicating blocked xor filter should be used.
27pub fn should_use_blocked_xor_filter_by_kv_count(
28    kv_count: u64,
29    blocked_kv_count_threshold: Option<u64>,
30) -> bool {
31    let threshold = blocked_kv_count_threshold
32        .unwrap_or(compaction_config::DEFAULT_BLOCKED_XOR_FILTER_KV_COUNT_THRESHOLD);
33    kv_count > threshold
34}
35
36pub fn parse_sstable_filter_kind(kind: &str) -> Result<PbSstableFilterType, String> {
37    match kind.trim().to_ascii_lowercase().as_str() {
38        "xor16" => Ok(PbSstableFilterType::SstableFilterXor16),
39        "xor8" => Ok(PbSstableFilterType::SstableFilterXor8),
40        _ => Err(format!("unsupported sstable filter kind: {kind}")),
41    }
42}
43
44pub fn parse_sstable_filter_layout(layout: &str) -> Result<PbSstableFilterLayout, String> {
45    match layout.trim().to_ascii_lowercase().as_str() {
46        "" | "auto" => Ok(PbSstableFilterLayout::Auto),
47        "plain" | "normal" | "nonblocked" | "non_blocked" | "non-blocked" => {
48            Ok(PbSstableFilterLayout::Plain)
49        }
50        "blocked" | "block" | "block_based" | "block-based" => Ok(PbSstableFilterLayout::Blocked),
51        _ => Err(format!("unsupported sstable filter layout: {layout}")),
52    }
53}
54
55pub fn get_sstable_filter_kind(
56    compaction_config: &CompactionConfig,
57    _base_level: usize,
58    level: usize,
59) -> Result<PbSstableFilterType, String> {
60    if compaction_config.sstable_filter_kind.is_empty() {
61        // Backward compatibility: old compaction configs did not carry sstable_filter_kind.
62        // Default to xor16 for all levels.
63        return Ok(PbSstableFilterType::SstableFilterXor16);
64    }
65
66    let raw_kind = compaction_config
67        .sstable_filter_kind
68        .get(level)
69        .ok_or_else(|| format!("sstable_filter_kind is not configured for level {level}"))?;
70
71    parse_sstable_filter_kind(raw_kind)
72}
73
74pub fn must_resolve_sstable_filter_kind(
75    compaction_config: &CompactionConfig,
76    base_level: usize,
77    level: usize,
78) -> PbSstableFilterType {
79    get_sstable_filter_kind(compaction_config, base_level, level)
80        .unwrap_or_else(|err| panic!("invalid sstable_filter_kind compaction config: {err}"))
81}
82
83pub fn get_sstable_filter_layout(
84    compaction_config: &CompactionConfig,
85    _base_level: usize,
86    level: usize,
87) -> Result<PbSstableFilterLayout, String> {
88    if compaction_config.sstable_filter_layout.is_empty() {
89        return Ok(PbSstableFilterLayout::Auto);
90    }
91
92    let raw_layout = compaction_config
93        .sstable_filter_layout
94        .get(level)
95        .ok_or_else(|| format!("sstable_filter_layout is not configured for level {level}"))?;
96
97    parse_sstable_filter_layout(raw_layout)
98}
99
100pub fn must_resolve_sstable_filter_layout(
101    compaction_config: &CompactionConfig,
102    base_level: usize,
103    level: usize,
104) -> PbSstableFilterLayout {
105    get_sstable_filter_layout(compaction_config, base_level, level)
106        .unwrap_or_else(|err| panic!("invalid sstable_filter_layout compaction config: {err}"))
107}
108
109#[cfg(test)]
110mod tests {
111    use risingwave_pb::hummock::CompactionConfig;
112
113    use super::{
114        PbSstableFilterLayout, PbSstableFilterType, get_sstable_filter_kind,
115        get_sstable_filter_layout, parse_sstable_filter_kind, parse_sstable_filter_layout,
116    };
117
118    #[test]
119    fn test_parse_sstable_filter_kind() {
120        assert_eq!(
121            parse_sstable_filter_kind("xor16").unwrap(),
122            PbSstableFilterType::SstableFilterXor16
123        );
124        assert_eq!(
125            parse_sstable_filter_kind("XOR8").unwrap(),
126            PbSstableFilterType::SstableFilterXor8
127        );
128        assert!(parse_sstable_filter_kind("bfuse8").is_err());
129    }
130
131    #[test]
132    fn test_parse_sstable_filter_layout() {
133        assert_eq!(
134            parse_sstable_filter_layout("auto").unwrap(),
135            PbSstableFilterLayout::Auto
136        );
137        assert_eq!(
138            parse_sstable_filter_layout("").unwrap(),
139            PbSstableFilterLayout::Auto
140        );
141        assert_eq!(
142            parse_sstable_filter_layout("plain").unwrap(),
143            PbSstableFilterLayout::Plain
144        );
145        assert_eq!(
146            parse_sstable_filter_layout("NORMAL").unwrap(),
147            PbSstableFilterLayout::Plain
148        );
149        assert_eq!(
150            parse_sstable_filter_layout("blocked").unwrap(),
151            PbSstableFilterLayout::Blocked
152        );
153        assert_eq!(
154            parse_sstable_filter_layout("block-based").unwrap(),
155            PbSstableFilterLayout::Blocked
156        );
157        assert!(parse_sstable_filter_layout("unknown").is_err());
158    }
159
160    #[test]
161    fn test_get_sstable_filter_kind_for_level() {
162        let config = CompactionConfig {
163            sstable_filter_kind: vec!["xor8".to_owned(), "xor16".to_owned(), "xor8".to_owned()],
164            ..Default::default()
165        };
166        assert_eq!(
167            get_sstable_filter_kind(&config, 2, 0).unwrap(),
168            PbSstableFilterType::SstableFilterXor8
169        );
170        assert_eq!(
171            get_sstable_filter_kind(&config, 2, 1).unwrap(),
172            PbSstableFilterType::SstableFilterXor16
173        );
174        assert_eq!(
175            get_sstable_filter_kind(&config, 2, 2).unwrap(),
176            PbSstableFilterType::SstableFilterXor8
177        );
178        assert!(get_sstable_filter_kind(&config, 2, 3).is_err());
179    }
180
181    #[test]
182    fn test_get_sstable_filter_kind_default_when_missing() {
183        let config = CompactionConfig {
184            sstable_filter_kind: vec![],
185            ..Default::default()
186        };
187        assert_eq!(
188            get_sstable_filter_kind(&config, 2, 0).unwrap(),
189            PbSstableFilterType::SstableFilterXor16
190        );
191        assert_eq!(
192            get_sstable_filter_kind(&config, 2, 6).unwrap(),
193            PbSstableFilterType::SstableFilterXor16
194        );
195    }
196
197    #[test]
198    fn test_get_sstable_filter_layout_for_level() {
199        let config = CompactionConfig {
200            sstable_filter_layout: vec![
201                "plain".to_owned(),
202                "auto".to_owned(),
203                "blocked".to_owned(),
204            ],
205            ..Default::default()
206        };
207        assert_eq!(
208            get_sstable_filter_layout(&config, 2, 0).unwrap(),
209            PbSstableFilterLayout::Plain
210        );
211        assert_eq!(
212            get_sstable_filter_layout(&config, 2, 1).unwrap(),
213            PbSstableFilterLayout::Auto
214        );
215        assert_eq!(
216            get_sstable_filter_layout(&config, 2, 2).unwrap(),
217            PbSstableFilterLayout::Blocked
218        );
219        assert!(get_sstable_filter_layout(&config, 2, 3).is_err());
220    }
221
222    #[test]
223    fn test_get_sstable_filter_layout_default_when_missing() {
224        let config = CompactionConfig {
225            sstable_filter_layout: vec![],
226            ..Default::default()
227        };
228        assert_eq!(
229            get_sstable_filter_layout(&config, 2, 0).unwrap(),
230            PbSstableFilterLayout::Auto
231        );
232        assert_eq!(
233            get_sstable_filter_layout(&config, 2, 6).unwrap(),
234            PbSstableFilterLayout::Auto
235        );
236    }
237}
risingwave_hummock_sdk/filter_utils.rs

risingwave_hummock_sdk/
filter_utils.rs
