risingwave_storage/
opts.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common::config::{
16    EvictionConfig, ObjectStoreConfig, RwConfig, StorageMemoryConfig, extract_storage_memory_config,
17};
18use risingwave_common::system_param::reader::{SystemParamsRead, SystemParamsReader};
19use risingwave_common::system_param::system_params_for_test;
20
21#[derive(Clone, Debug)]
22pub struct StorageOpts {
23    /// The size of parallel task for one compact/flush job.
24    pub parallel_compact_size_mb: u32,
25    /// Target size of the Sstable.
26    pub sstable_size_mb: u32,
27    /// Minimal target size of the Sstable to store data of different state-table in independent files as soon as possible.
28    pub min_sstable_size_mb: u32,
29    /// Size of each block in bytes in SST.
30    pub block_size_kb: u32,
31    /// False positive probability of bloom filter.
32    pub bloom_false_positive: f64,
33    /// parallelism while syncing share buffers into L0 SST. Should NOT be 0.
34    pub share_buffers_sync_parallelism: u32,
35    /// Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use
36    /// tokio's default value (number of CPU core).
37    pub share_buffer_compaction_worker_threads_number: u32,
38    /// Maximum shared buffer size, writes attempting to exceed the capacity will stall until there
39    /// is enough space.
40    pub shared_buffer_capacity_mb: usize,
41    /// The shared buffer will start flushing data to object when the ratio of memory usage to the
42    /// shared buffer capacity exceed such ratio.
43    pub shared_buffer_flush_ratio: f32,
44    /// The minimum total flush size of shared buffer spill. When a shared buffer spill is trigger,
45    /// the total flush size across multiple epochs should be at least higher than this size.
46    pub shared_buffer_min_batch_flush_size_mb: usize,
47    /// Remote directory for storing data and metadata objects.
48    pub data_directory: String,
49    /// Whether to enable write conflict detection
50    pub write_conflict_detection_enabled: bool,
51    /// Capacity of sstable block cache.
52    pub block_cache_capacity_mb: usize,
53    /// the number of block-cache shard. Less shard means that more concurrent-conflict.
54    pub block_cache_shard_num: usize,
55    /// Eviction config for block cache.
56    pub block_cache_eviction_config: EvictionConfig,
57    /// Capacity of sstable meta cache.
58    pub meta_cache_capacity_mb: usize,
59    /// the number of meta-cache shard. Less shard means that more concurrent-conflict.
60    pub meta_cache_shard_num: usize,
61    /// Eviction config for meta cache.
62    pub meta_cache_eviction_config: EvictionConfig,
63    /// max memory usage for large query.
64    pub prefetch_buffer_capacity_mb: usize,
65
66    pub max_cached_recent_versions_number: usize,
67
68    pub max_prefetch_block_number: usize,
69
70    pub disable_remote_compactor: bool,
71    /// Number of tasks shared buffer can upload in parallel.
72    pub share_buffer_upload_concurrency: usize,
73    /// Capacity of sstable meta cache.
74    pub compactor_memory_limit_mb: usize,
75    /// compactor streaming iterator recreate timeout.
76    /// deprecated
77    pub compact_iter_recreate_timeout_ms: u64,
78    /// Number of SST ids fetched from meta per RPC
79    pub sstable_id_remote_fetch_number: u32,
80    /// Whether to enable streaming upload for sstable.
81    pub min_sst_size_for_streaming_upload: u64,
82    pub max_concurrent_compaction_task_number: u64,
83    pub max_version_pinning_duration_sec: u64,
84    pub compactor_iter_max_io_retry_times: usize,
85
86    pub data_file_cache_dir: String,
87    pub data_file_cache_capacity_mb: usize,
88    pub data_file_cache_file_capacity_mb: usize,
89    pub data_file_cache_flushers: usize,
90    pub data_file_cache_reclaimers: usize,
91    pub data_file_cache_recover_mode: foyer::RecoverMode,
92    pub data_file_cache_recover_concurrency: usize,
93    pub data_file_cache_indexer_shards: usize,
94    pub data_file_cache_compression: foyer::Compression,
95    pub data_file_cache_flush_buffer_threshold_mb: usize,
96    pub data_file_cache_fifo_probation_ratio: f64,
97    pub data_file_cache_blob_index_size_kb: usize,
98    pub data_file_cache_runtime_config: foyer::RuntimeOptions,
99    pub data_file_cache_throttle: foyer::Throttle,
100
101    pub cache_refill_data_refill_levels: Vec<u32>,
102    pub cache_refill_timeout_ms: u64,
103    pub cache_refill_concurrency: usize,
104    pub cache_refill_recent_filter_shards: usize,
105    pub cache_refill_recent_filter_layers: usize,
106    pub cache_refill_recent_filter_rotate_interval_ms: usize,
107    pub cache_refill_unit: usize,
108    pub cache_refill_threshold: f64,
109    pub cache_refill_skip_recent_filter: bool,
110
111    pub meta_file_cache_dir: String,
112    pub meta_file_cache_capacity_mb: usize,
113    pub meta_file_cache_file_capacity_mb: usize,
114    pub meta_file_cache_flushers: usize,
115    pub meta_file_cache_reclaimers: usize,
116    pub meta_file_cache_recover_mode: foyer::RecoverMode,
117    pub meta_file_cache_recover_concurrency: usize,
118    pub meta_file_cache_indexer_shards: usize,
119    pub meta_file_cache_compression: foyer::Compression,
120    pub meta_file_cache_flush_buffer_threshold_mb: usize,
121    pub meta_file_cache_fifo_probation_ratio: f64,
122    pub meta_file_cache_blob_index_size_kb: usize,
123    pub meta_file_cache_runtime_config: foyer::RuntimeOptions,
124    pub meta_file_cache_throttle: foyer::Throttle,
125
126    pub vector_file_block_size_kb: usize,
127    pub vector_block_cache_capacity_mb: usize,
128    pub vector_block_cache_shard_num: usize,
129    pub vector_block_cache_eviction_config: EvictionConfig,
130    pub vector_meta_cache_capacity_mb: usize,
131    pub vector_meta_cache_shard_num: usize,
132    pub vector_meta_cache_eviction_config: EvictionConfig,
133
134    /// The storage url for storing backups.
135    pub backup_storage_url: String,
136    /// The storage directory for storing backups.
137    pub backup_storage_directory: String,
138    /// max time which wait for preload. 0 represent do not do any preload.
139    pub max_preload_wait_time_mill: u64,
140
141    pub compactor_max_sst_key_count: u64,
142    pub compactor_max_task_multiplier: f32,
143    pub compactor_max_sst_size: u64,
144    /// enable `FastCompactorRunner`.
145    pub enable_fast_compaction: bool,
146    pub check_compaction_result: bool,
147    pub max_preload_io_retry_times: usize,
148    pub compactor_fast_max_compact_delete_ratio: u32,
149    pub compactor_fast_max_compact_task_size: u64,
150
151    pub mem_table_spill_threshold: usize,
152
153    pub compactor_concurrent_uploading_sst_count: Option<usize>,
154
155    pub compactor_max_overlap_sst_count: usize,
156
157    /// The maximum number of meta files that can be preloaded.
158    pub compactor_max_preload_meta_file_count: usize,
159
160    pub object_store_config: ObjectStoreConfig,
161    pub time_travel_version_cache_capacity: u64,
162
163    pub iceberg_compaction_target_file_size_mb: u32,
164    pub iceberg_compaction_enable_validate: bool,
165    pub iceberg_compaction_max_record_batch_rows: usize,
166    pub iceberg_compaction_write_parquet_max_row_group_rows: usize,
167    pub iceberg_compaction_min_size_per_partition_mb: u32,
168    pub iceberg_compaction_max_file_count_per_partition: u32,
169    pub iceberg_compaction_small_file_threshold_mb: u32,
170    pub iceberg_compaction_max_task_total_size_mb: u32,
171
172    /// The ratio of iceberg compaction max parallelism to the number of CPU cores
173    pub iceberg_compaction_task_parallelism_ratio: f32,
174    /// Whether to enable heuristic output parallelism in iceberg compaction.
175    pub iceberg_compaction_enable_heuristic_output_parallelism: bool,
176    /// Maximum number of concurrent file close operations
177    pub iceberg_compaction_max_concurrent_closes: usize,
178    /// Whether to enable dynamic size estimation for iceberg compaction.
179    pub iceberg_compaction_enable_dynamic_size_estimation: bool,
180    /// The smoothing factor for size estimation in iceberg compaction.(default: 0.3)
181    pub iceberg_compaction_size_estimation_smoothing_factor: f64,
182}
183
184impl Default for StorageOpts {
185    fn default() -> Self {
186        let c = RwConfig::default();
187        let p = system_params_for_test();
188        let s = extract_storage_memory_config(&c);
189        Self::from((&c, &p.into(), &s))
190    }
191}
192
193impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpts {
194    fn from((c, p, s): (&RwConfig, &SystemParamsReader, &StorageMemoryConfig)) -> Self {
195        let mut data_file_cache_throttle = c.storage.data_file_cache.throttle.clone();
196        if data_file_cache_throttle.write_throughput.is_none() {
197            data_file_cache_throttle = data_file_cache_throttle.with_write_throughput(
198                c.storage.data_file_cache.insert_rate_limit_mb * 1024 * 1024,
199            );
200        }
201        let mut meta_file_cache_throttle = c.storage.meta_file_cache.throttle.clone();
202        if meta_file_cache_throttle.write_throughput.is_none() {
203            meta_file_cache_throttle = meta_file_cache_throttle.with_write_throughput(
204                c.storage.meta_file_cache.insert_rate_limit_mb * 1024 * 1024,
205            );
206        }
207
208        Self {
209            parallel_compact_size_mb: p.parallel_compact_size_mb(),
210            sstable_size_mb: p.sstable_size_mb(),
211            min_sstable_size_mb: c.storage.min_sstable_size_mb,
212            block_size_kb: p.block_size_kb(),
213            bloom_false_positive: p.bloom_false_positive(),
214            share_buffers_sync_parallelism: c.storage.share_buffers_sync_parallelism,
215            share_buffer_compaction_worker_threads_number: c
216                .storage
217                .share_buffer_compaction_worker_threads_number,
218            shared_buffer_capacity_mb: s.shared_buffer_capacity_mb,
219            shared_buffer_flush_ratio: c.storage.shared_buffer_flush_ratio,
220            shared_buffer_min_batch_flush_size_mb: c.storage.shared_buffer_min_batch_flush_size_mb,
221            data_directory: p.data_directory().to_owned(),
222            write_conflict_detection_enabled: c.storage.write_conflict_detection_enabled,
223            block_cache_capacity_mb: s.block_cache_capacity_mb,
224            block_cache_shard_num: s.block_cache_shard_num,
225            block_cache_eviction_config: s.block_cache_eviction_config.clone(),
226            meta_cache_capacity_mb: s.meta_cache_capacity_mb,
227            meta_cache_shard_num: s.meta_cache_shard_num,
228            meta_cache_eviction_config: s.meta_cache_eviction_config.clone(),
229            prefetch_buffer_capacity_mb: s.prefetch_buffer_capacity_mb,
230            max_cached_recent_versions_number: c.storage.max_cached_recent_versions_number,
231            max_prefetch_block_number: c.storage.max_prefetch_block_number,
232            disable_remote_compactor: c.storage.disable_remote_compactor,
233            share_buffer_upload_concurrency: c.storage.share_buffer_upload_concurrency,
234            compactor_memory_limit_mb: s.compactor_memory_limit_mb,
235            sstable_id_remote_fetch_number: c.storage.sstable_id_remote_fetch_number,
236            min_sst_size_for_streaming_upload: c.storage.min_sst_size_for_streaming_upload,
237            max_concurrent_compaction_task_number: c.storage.max_concurrent_compaction_task_number,
238            max_version_pinning_duration_sec: c.storage.max_version_pinning_duration_sec,
239            data_file_cache_dir: c.storage.data_file_cache.dir.clone(),
240            data_file_cache_capacity_mb: c.storage.data_file_cache.capacity_mb,
241            data_file_cache_file_capacity_mb: c.storage.data_file_cache.file_capacity_mb,
242            data_file_cache_flushers: c.storage.data_file_cache.flushers,
243            data_file_cache_reclaimers: c.storage.data_file_cache.reclaimers,
244            data_file_cache_recover_mode: c.storage.data_file_cache.recover_mode,
245            data_file_cache_recover_concurrency: c.storage.data_file_cache.recover_concurrency,
246            data_file_cache_indexer_shards: c.storage.data_file_cache.indexer_shards,
247            data_file_cache_compression: c.storage.data_file_cache.compression,
248            data_file_cache_flush_buffer_threshold_mb: s.block_file_cache_flush_buffer_threshold_mb,
249            data_file_cache_fifo_probation_ratio: c.storage.data_file_cache.fifo_probation_ratio,
250            data_file_cache_blob_index_size_kb: c.storage.data_file_cache.blob_index_size_kb,
251            data_file_cache_runtime_config: c.storage.data_file_cache.runtime_config.clone(),
252            data_file_cache_throttle,
253            meta_file_cache_dir: c.storage.meta_file_cache.dir.clone(),
254            meta_file_cache_capacity_mb: c.storage.meta_file_cache.capacity_mb,
255            meta_file_cache_file_capacity_mb: c.storage.meta_file_cache.file_capacity_mb,
256            meta_file_cache_flushers: c.storage.meta_file_cache.flushers,
257            meta_file_cache_reclaimers: c.storage.meta_file_cache.reclaimers,
258            meta_file_cache_recover_mode: c.storage.meta_file_cache.recover_mode,
259            meta_file_cache_recover_concurrency: c.storage.meta_file_cache.recover_concurrency,
260            meta_file_cache_indexer_shards: c.storage.meta_file_cache.indexer_shards,
261            meta_file_cache_compression: c.storage.meta_file_cache.compression,
262            meta_file_cache_flush_buffer_threshold_mb: s.meta_file_cache_flush_buffer_threshold_mb,
263            meta_file_cache_fifo_probation_ratio: c.storage.meta_file_cache.fifo_probation_ratio,
264            meta_file_cache_blob_index_size_kb: c.storage.meta_file_cache.blob_index_size_kb,
265            meta_file_cache_runtime_config: c.storage.meta_file_cache.runtime_config.clone(),
266            meta_file_cache_throttle,
267            cache_refill_data_refill_levels: c.storage.cache_refill.data_refill_levels.clone(),
268            cache_refill_timeout_ms: c.storage.cache_refill.timeout_ms,
269            cache_refill_concurrency: c.storage.cache_refill.concurrency,
270            cache_refill_recent_filter_shards: c.storage.cache_refill.recent_filter_shards,
271            cache_refill_recent_filter_layers: c.storage.cache_refill.recent_filter_layers,
272            cache_refill_recent_filter_rotate_interval_ms: c
273                .storage
274                .cache_refill
275                .recent_filter_rotate_interval_ms,
276            cache_refill_unit: c.storage.cache_refill.unit,
277            cache_refill_threshold: c.storage.cache_refill.threshold,
278            cache_refill_skip_recent_filter: c.storage.cache_refill.skip_recent_filter,
279            max_preload_wait_time_mill: c.storage.max_preload_wait_time_mill,
280            compact_iter_recreate_timeout_ms: c.storage.compact_iter_recreate_timeout_ms,
281
282            max_preload_io_retry_times: c.storage.max_preload_io_retry_times,
283            backup_storage_url: p.backup_storage_url().to_owned(),
284            backup_storage_directory: p.backup_storage_directory().to_owned(),
285            compactor_max_sst_key_count: c.storage.compactor_max_sst_key_count,
286            compactor_max_task_multiplier: c.storage.compactor_max_task_multiplier,
287            compactor_max_sst_size: c.storage.compactor_max_sst_size,
288            enable_fast_compaction: c.storage.enable_fast_compaction,
289            check_compaction_result: c.storage.check_compaction_result,
290            mem_table_spill_threshold: c.storage.mem_table_spill_threshold,
291            object_store_config: c.storage.object_store.clone(),
292            compactor_fast_max_compact_delete_ratio: c
293                .storage
294                .compactor_fast_max_compact_delete_ratio,
295            compactor_fast_max_compact_task_size: c.storage.compactor_fast_max_compact_task_size,
296            compactor_iter_max_io_retry_times: c.storage.compactor_iter_max_io_retry_times,
297            compactor_concurrent_uploading_sst_count: c
298                .storage
299                .compactor_concurrent_uploading_sst_count,
300            time_travel_version_cache_capacity: c.storage.time_travel_version_cache_capacity,
301            compactor_max_overlap_sst_count: c.storage.compactor_max_overlap_sst_count,
302            compactor_max_preload_meta_file_count: c.storage.compactor_max_preload_meta_file_count,
303
304            iceberg_compaction_target_file_size_mb: c
305                .storage
306                .iceberg_compaction_target_file_size_mb,
307            iceberg_compaction_enable_validate: c.storage.iceberg_compaction_enable_validate,
308            iceberg_compaction_max_record_batch_rows: c
309                .storage
310                .iceberg_compaction_max_record_batch_rows,
311            iceberg_compaction_write_parquet_max_row_group_rows: c
312                .storage
313                .iceberg_compaction_write_parquet_max_row_group_rows,
314            iceberg_compaction_min_size_per_partition_mb: c
315                .storage
316                .iceberg_compaction_min_size_per_partition_mb,
317            iceberg_compaction_max_file_count_per_partition: c
318                .storage
319                .iceberg_compaction_max_file_count_per_partition,
320            iceberg_compaction_small_file_threshold_mb: c
321                .storage
322                .iceberg_compaction_small_file_threshold_mb,
323            iceberg_compaction_max_task_total_size_mb: c
324                .storage
325                .iceberg_compaction_max_task_total_size_mb,
326            iceberg_compaction_task_parallelism_ratio: c
327                .storage
328                .iceberg_compaction_task_parallelism_ratio,
329            iceberg_compaction_enable_heuristic_output_parallelism: c
330                .storage
331                .iceberg_compaction_enable_heuristic_output_parallelism,
332            iceberg_compaction_max_concurrent_closes: c
333                .storage
334                .iceberg_compaction_max_concurrent_closes,
335            iceberg_compaction_enable_dynamic_size_estimation: c
336                .storage
337                .iceberg_compaction_enable_dynamic_size_estimation,
338            iceberg_compaction_size_estimation_smoothing_factor: c
339                .storage
340                .iceberg_compaction_size_estimation_smoothing_factor,
341            vector_file_block_size_kb: c.storage.vector_file_block_size_kb,
342            vector_block_cache_capacity_mb: s.vector_block_cache_capacity_mb,
343            vector_block_cache_shard_num: s.vector_block_cache_shard_num,
344            vector_block_cache_eviction_config: s.vector_block_cache_eviction_config.clone(),
345            vector_meta_cache_capacity_mb: s.vector_meta_cache_capacity_mb,
346            vector_meta_cache_shard_num: s.vector_meta_cache_shard_num,
347            vector_meta_cache_eviction_config: s.vector_meta_cache_eviction_config.clone(),
348        }
349    }
350}
risingwave_storage/opts.rs

risingwave_storage/
opts.rs
