risingwave_hummock_sdk/
filter_utils.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common::config::meta::default::compaction_config;
16use risingwave_pb::hummock::{CompactionConfig, PbSstableFilterLayout, PbSstableFilterType};
17
18/// Determines whether the key count is large enough to warrant using a blocked xor filter.
19///
20/// # Arguments
21/// * `kv_count` - The total number of keys
22/// * `blocked_kv_count_threshold` - Optional configured threshold. If None, uses
23///   `DEFAULT_BLOCKED_XOR_FILTER_KV_COUNT_THRESHOLD`.
24///
25/// # Returns
26/// `true` if `kv_count` exceeds the threshold, indicating blocked xor filter should be used.
27pub fn should_use_blocked_xor_filter_by_kv_count(
28    kv_count: u64,
29    blocked_kv_count_threshold: Option<u64>,
30) -> bool {
31    let threshold = blocked_kv_count_threshold
32        .unwrap_or(compaction_config::DEFAULT_BLOCKED_XOR_FILTER_KV_COUNT_THRESHOLD);
33    kv_count > threshold
34}
35
36pub fn parse_sstable_filter_type(filter_type: &str) -> Result<PbSstableFilterType, String> {
37    match filter_type {
38        "none" => Ok(PbSstableFilterType::SstableFilterNone),
39        "xor16" => Ok(PbSstableFilterType::SstableFilterXor16),
40        "xor8" => Ok(PbSstableFilterType::SstableFilterXor8),
41        _ => Err(format!("unsupported sstable filter type: {filter_type}")),
42    }
43}
44
45pub fn parse_sstable_filter_layout(layout: &str) -> Result<PbSstableFilterLayout, String> {
46    match layout {
47        "auto" => Ok(PbSstableFilterLayout::Auto),
48        "plain" => Ok(PbSstableFilterLayout::Plain),
49        "blocked" => Ok(PbSstableFilterLayout::Blocked),
50        _ => Err(format!("unsupported sstable filter layout: {layout}")),
51    }
52}
53
54pub fn get_sstable_filter_type(
55    compaction_config: &CompactionConfig,
56    _base_level: usize,
57    level: usize,
58) -> Result<PbSstableFilterType, String> {
59    if compaction_config.sstable_filter_type.is_empty() {
60        // Backward compatibility: old compaction configs did not carry the filter type field.
61        // Default to xor16 for all levels.
62        return Ok(PbSstableFilterType::SstableFilterXor16);
63    }
64
65    let raw_filter_type = compaction_config
66        .sstable_filter_type
67        .get(level)
68        .ok_or_else(|| format!("sstable_filter_type is not configured for level {level}"))?;
69
70    parse_sstable_filter_type(raw_filter_type)
71}
72
73pub fn must_resolve_sstable_filter_type(
74    compaction_config: &CompactionConfig,
75    base_level: usize,
76    level: usize,
77) -> PbSstableFilterType {
78    get_sstable_filter_type(compaction_config, base_level, level)
79        .unwrap_or_else(|err| panic!("invalid sstable_filter_type compaction config: {err}"))
80}
81
82pub fn get_sstable_filter_layout(
83    compaction_config: &CompactionConfig,
84    _base_level: usize,
85    level: usize,
86) -> Result<PbSstableFilterLayout, String> {
87    if compaction_config.sstable_filter_layout.is_empty() {
88        return Ok(PbSstableFilterLayout::Auto);
89    }
90
91    let raw_layout = compaction_config
92        .sstable_filter_layout
93        .get(level)
94        .ok_or_else(|| format!("sstable_filter_layout is not configured for level {level}"))?;
95
96    parse_sstable_filter_layout(raw_layout)
97}
98
99pub fn must_resolve_sstable_filter_layout(
100    compaction_config: &CompactionConfig,
101    base_level: usize,
102    level: usize,
103) -> PbSstableFilterLayout {
104    get_sstable_filter_layout(compaction_config, base_level, level)
105        .unwrap_or_else(|err| panic!("invalid sstable_filter_layout compaction config: {err}"))
106}
107
108#[cfg(test)]
109mod tests {
110    use risingwave_pb::hummock::CompactionConfig;
111
112    use super::{
113        PbSstableFilterLayout, PbSstableFilterType, get_sstable_filter_layout,
114        get_sstable_filter_type, parse_sstable_filter_layout, parse_sstable_filter_type,
115    };
116
117    #[test]
118    fn test_parse_sstable_filter_type() {
119        assert_eq!(
120            parse_sstable_filter_type("none").unwrap(),
121            PbSstableFilterType::SstableFilterNone
122        );
123        assert_eq!(
124            parse_sstable_filter_type("xor16").unwrap(),
125            PbSstableFilterType::SstableFilterXor16
126        );
127        assert_eq!(
128            parse_sstable_filter_type("xor8").unwrap(),
129            PbSstableFilterType::SstableFilterXor8
130        );
131        assert!(parse_sstable_filter_type("XOR8").is_err());
132        assert!(parse_sstable_filter_type("bfuse8").is_err());
133    }
134
135    #[test]
136    fn test_parse_sstable_filter_layout() {
137        assert_eq!(
138            parse_sstable_filter_layout("auto").unwrap(),
139            PbSstableFilterLayout::Auto
140        );
141        assert_eq!(
142            parse_sstable_filter_layout("plain").unwrap(),
143            PbSstableFilterLayout::Plain
144        );
145        assert_eq!(
146            parse_sstable_filter_layout("blocked").unwrap(),
147            PbSstableFilterLayout::Blocked
148        );
149        assert!(parse_sstable_filter_layout("blocked ").is_err());
150        assert!(parse_sstable_filter_layout("none").is_err());
151        assert!(parse_sstable_filter_layout("unknown").is_err());
152    }
153
154    #[test]
155    fn test_get_sstable_filter_type() {
156        let config = CompactionConfig {
157            sstable_filter_type: vec!["xor8".to_owned(), "none".to_owned(), "xor16".to_owned()],
158            ..Default::default()
159        };
160        assert_eq!(
161            get_sstable_filter_type(&config, 2, 0).unwrap(),
162            PbSstableFilterType::SstableFilterXor8
163        );
164        assert_eq!(
165            get_sstable_filter_type(&config, 2, 1).unwrap(),
166            PbSstableFilterType::SstableFilterNone
167        );
168        assert_eq!(
169            get_sstable_filter_type(&config, 2, 2).unwrap(),
170            PbSstableFilterType::SstableFilterXor16
171        );
172        assert!(get_sstable_filter_type(&config, 2, 3).is_err());
173
174        let config = CompactionConfig {
175            sstable_filter_type: vec![],
176            ..Default::default()
177        };
178        assert_eq!(
179            get_sstable_filter_type(&config, 2, 0).unwrap(),
180            PbSstableFilterType::SstableFilterXor16
181        );
182        assert_eq!(
183            get_sstable_filter_type(&config, 2, 6).unwrap(),
184            PbSstableFilterType::SstableFilterXor16
185        );
186    }
187
188    #[test]
189    fn test_get_sstable_filter_layout() {
190        let config = CompactionConfig {
191            sstable_filter_layout: vec![
192                "plain".to_owned(),
193                "auto".to_owned(),
194                "blocked".to_owned(),
195            ],
196            ..Default::default()
197        };
198        assert_eq!(
199            get_sstable_filter_layout(&config, 2, 0).unwrap(),
200            PbSstableFilterLayout::Plain
201        );
202        assert_eq!(
203            get_sstable_filter_layout(&config, 2, 1).unwrap(),
204            PbSstableFilterLayout::Auto
205        );
206        assert_eq!(
207            get_sstable_filter_layout(&config, 2, 2).unwrap(),
208            PbSstableFilterLayout::Blocked
209        );
210        assert!(get_sstable_filter_layout(&config, 2, 3).is_err());
211
212        let config = CompactionConfig {
213            sstable_filter_layout: vec![],
214            ..Default::default()
215        };
216        assert_eq!(
217            get_sstable_filter_layout(&config, 2, 0).unwrap(),
218            PbSstableFilterLayout::Auto
219        );
220        assert_eq!(
221            get_sstable_filter_layout(&config, 2, 6).unwrap(),
222            PbSstableFilterLayout::Auto
223        );
224    }
225}
risingwave_hummock_sdk/filter_utils.rs

risingwave_hummock_sdk/
filter_utils.rs
