risingwave_common/config/
storage.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use foyer::{Compression, LfuConfig, LruConfig, RecoverMode, S3FifoConfig, Throttle};
16use serde::de::Error as _;
17
18use super::*;
19
20/// The section `[storage]` in `risingwave.toml`.
21#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
22#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
23pub struct StorageConfig {
24    /// parallelism while syncing share buffers into L0 SST. Should NOT be 0.
25    #[serde(default = "default::storage::share_buffers_sync_parallelism")]
26    pub share_buffers_sync_parallelism: u32,
27
28    /// Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use
29    /// tokio's default value (number of CPU core).
30    #[serde(default = "default::storage::share_buffer_compaction_worker_threads_number")]
31    pub share_buffer_compaction_worker_threads_number: u32,
32
33    /// Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity
34    /// will stall until there is enough space. The overridden value will only be effective if:
35    /// 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly.
36    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
37    #[serde(default)]
38    pub shared_buffer_capacity_mb: Option<usize>,
39
40    /// The shared buffer will start flushing data to object when the ratio of memory usage to the
41    /// shared buffer capacity exceed such ratio.
42    #[serde(default = "default::storage::shared_buffer_flush_ratio")]
43    pub shared_buffer_flush_ratio: f32,
44
45    /// The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger,
46    /// the total flush size across multiple epochs should be at least higher than this size.
47    #[serde(default = "default::storage::shared_buffer_min_batch_flush_size_mb")]
48    pub shared_buffer_min_batch_flush_size_mb: usize,
49
50    /// The threshold for the number of immutable memtables to merge to a new imm.
51    #[serde(default = "default::storage::imm_merge_threshold")]
52    #[deprecated]
53    pub imm_merge_threshold: usize,
54
55    /// Whether to enable write conflict detection
56    #[serde(default = "default::storage::write_conflict_detection_enabled")]
57    pub write_conflict_detection_enabled: bool,
58
59    #[serde(default)]
60    #[config_doc(nested)]
61    pub cache: CacheConfig,
62
63    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead.
64    #[serde(default)]
65    pub block_cache_capacity_mb: Option<usize>,
66
67    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead.
68    #[serde(default)]
69    pub meta_cache_capacity_mb: Option<usize>,
70
71    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead.
72    #[serde(default)]
73    pub high_priority_ratio_in_percent: Option<usize>,
74
75    /// max memory usage for large query
76    #[serde(default)]
77    pub prefetch_buffer_capacity_mb: Option<usize>,
78
79    #[serde(default = "default::storage::max_cached_recent_versions_number")]
80    pub max_cached_recent_versions_number: usize,
81
82    /// max prefetch block number
83    #[serde(
84        default = "default::storage::max_prefetch_block_number",
85        deserialize_with = "deserialize_max_prefetch_block_number"
86    )]
87    pub max_prefetch_block_number: usize,
88
89    #[serde(default = "default::storage::disable_remote_compactor")]
90    pub disable_remote_compactor: bool,
91
92    /// Number of tasks shared buffer can upload in parallel.
93    #[serde(default = "default::storage::share_buffer_upload_concurrency")]
94    pub share_buffer_upload_concurrency: usize,
95
96    #[serde(default)]
97    pub compactor_memory_limit_mb: Option<usize>,
98
99    /// Compactor calculates the maximum number of tasks that can be executed on the node based on
100    /// `worker_num` and `compactor_max_task_multiplier`.
101    /// `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier`
102    #[serde(default = "default::storage::compactor_max_task_multiplier")]
103    pub compactor_max_task_multiplier: f32,
104
105    /// The percentage of memory available when compactor is deployed separately.
106    /// `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion`
107    #[serde(default = "default::storage::compactor_memory_available_proportion")]
108    pub compactor_memory_available_proportion: f64,
109
110    /// Number of SST ids fetched from meta per RPC
111    #[serde(default = "default::storage::sstable_id_remote_fetch_number")]
112    pub sstable_id_remote_fetch_number: u32,
113
114    #[serde(default = "default::storage::min_sstable_size_mb")]
115    pub min_sstable_size_mb: u32,
116
117    #[serde(default)]
118    #[config_doc(nested)]
119    pub data_file_cache: FileCacheConfig,
120
121    #[serde(default)]
122    #[config_doc(nested)]
123    pub meta_file_cache: FileCacheConfig,
124
125    /// sst serde happens when a sst meta is written to meta disk cache.
126    /// Excluding the SST filter from serde can reduce the meta disk cache entry size
127    /// and reduce disk IO throughput at the cost of making the SST filter useless.
128    #[serde(default = "default::storage::sst_skip_bloom_filter_in_serde")]
129    pub sst_skip_bloom_filter_in_serde: bool,
130
131    #[serde(default)]
132    #[config_doc(nested)]
133    pub cache_refill: CacheRefillConfig,
134
135    /// Whether to enable streaming upload for sstable.
136    #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
137    pub min_sst_size_for_streaming_upload: u64,
138
139    #[serde(default = "default::storage::max_concurrent_compaction_task_number")]
140    pub max_concurrent_compaction_task_number: u64,
141
142    #[serde(default = "default::storage::max_preload_wait_time_mill")]
143    pub max_preload_wait_time_mill: u64,
144
145    #[serde(default = "default::storage::max_version_pinning_duration_sec")]
146    pub max_version_pinning_duration_sec: u64,
147
148    #[serde(default = "default::storage::compactor_max_sst_key_count")]
149    pub compactor_max_sst_key_count: u64,
150    // DEPRECATED: This config will be deprecated in the future version, use `storage.compactor_iter_max_io_retry_times` instead.
151    #[serde(default = "default::storage::compact_iter_recreate_timeout_ms")]
152    pub compact_iter_recreate_timeout_ms: u64,
153    #[serde(default = "default::storage::compactor_max_sst_size")]
154    pub compactor_max_sst_size: u64,
155    #[serde(default = "default::storage::enable_fast_compaction")]
156    pub enable_fast_compaction: bool,
157    #[serde(default = "default::storage::check_compaction_result")]
158    pub check_compaction_result: bool,
159    /// Legacy setting retained for configuration compatibility. This has no effect because
160    /// prefetched blocks are fully buffered before consumption.
161    #[serde(default = "default::storage::max_preload_io_retry_times")]
162    pub max_preload_io_retry_times: usize,
163    #[serde(default = "default::storage::compactor_fast_max_compact_delete_ratio")]
164    pub compactor_fast_max_compact_delete_ratio: u32,
165    #[serde(default = "default::storage::compactor_fast_max_compact_task_size")]
166    pub compactor_fast_max_compact_task_size: u64,
167    #[serde(default = "default::storage::compactor_iter_max_io_retry_times")]
168    pub compactor_iter_max_io_retry_times: usize,
169
170    /// If set, block metadata keys will be shortened when their length exceeds this threshold.
171    /// This reduces `SSTable` metadata size by storing only the minimal distinguishing prefix.
172    /// - `None`: Disabled (default)
173    /// - `Some(n)`: Only shorten keys with length >= n bytes
174    #[serde(default = "default::storage::shorten_block_meta_key_threshold")]
175    pub shorten_block_meta_key_threshold: Option<usize>,
176
177    /// Deprecated: The window size of table info statistic history.
178    #[serde(default = "default::storage::table_info_statistic_history_times")]
179    #[deprecated]
180    pub table_info_statistic_history_times: usize,
181
182    #[serde(default, flatten)]
183    #[config_doc(omitted)]
184    pub unrecognized: Unrecognized<Self>,
185
186    /// The spill threshold for mem table.
187    #[serde(default = "default::storage::mem_table_spill_threshold")]
188    pub mem_table_spill_threshold: usize,
189
190    /// The concurrent uploading number of `SSTables` of builder
191    #[serde(default = "default::storage::compactor_concurrent_uploading_sst_count")]
192    pub compactor_concurrent_uploading_sst_count: Option<usize>,
193
194    #[serde(default = "default::storage::compactor_max_overlap_sst_count")]
195    pub compactor_max_overlap_sst_count: usize,
196
197    /// The maximum number of meta files that can be preloaded.
198    /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`.
199    /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient.
200    #[serde(default = "default::storage::compactor_max_preload_meta_file_count")]
201    pub compactor_max_preload_meta_file_count: usize,
202
203    #[serde(default = "default::storage::vector_file_block_size_kb")]
204    pub vector_file_block_size_kb: usize,
205
206    /// Object storage configuration
207    /// 1. General configuration
208    /// 2. Some special configuration of Backend
209    /// 3. Retry and timeout configuration
210    #[serde(default)]
211    pub object_store: ObjectStoreConfig,
212
213    #[serde(default = "default::storage::time_travel_version_cache_capacity")]
214    pub time_travel_version_cache_capacity: u64,
215
216    #[serde(default = "default::storage::table_change_log_cache_capacity")]
217    pub table_change_log_cache_capacity: u64,
218
219    // iceberg compaction
220    /// Estimated heap memory budget used to schedule tasks in the dedicated Iceberg compactor, in
221    /// megabytes. This controls admission only; it is not a hard `DataFusion` allocation limit.
222    /// When unset, the budget is derived from the compactor's available memory.
223    #[serde(default)]
224    pub iceberg_compaction_memory_limit_mb: Option<usize>,
225    #[serde(default = "default::storage::iceberg_compaction_enable_validate")]
226    pub iceberg_compaction_enable_validate: bool,
227    #[serde(default = "default::storage::iceberg_compaction_max_record_batch_rows")]
228    pub iceberg_compaction_max_record_batch_rows: usize,
229    #[serde(default = "default::storage::iceberg_compaction_min_size_per_partition_mb")]
230    pub iceberg_compaction_min_size_per_partition_mb: u32,
231    #[serde(default = "default::storage::iceberg_compaction_max_file_count_per_partition")]
232    pub iceberg_compaction_max_file_count_per_partition: u32,
233    /// DEPRECATED: This config will be deprecated in the future version.
234    /// Use sink config `compaction.write_parquet_max_row_group_rows` instead.
235    #[serde(default = "default::storage::iceberg_compaction_write_parquet_max_row_group_rows")]
236    #[deprecated(
237        note = "This config is deprecated. Use sink config `compaction.write_parquet_max_row_group_rows` instead."
238    )]
239    pub iceberg_compaction_write_parquet_max_row_group_rows: usize,
240
241    /// The ratio of iceberg compaction max parallelism to the number of CPU cores
242    #[serde(default = "default::storage::iceberg_compaction_task_parallelism_ratio")]
243    pub iceberg_compaction_task_parallelism_ratio: f32,
244    /// Whether to enable heuristic output parallelism in iceberg compaction.
245    #[serde(default = "default::storage::iceberg_compaction_enable_heuristic_output_parallelism")]
246    pub iceberg_compaction_enable_heuristic_output_parallelism: bool,
247    /// Maximum number of concurrent file close operations
248    #[serde(default = "default::storage::iceberg_compaction_max_concurrent_closes")]
249    pub iceberg_compaction_max_concurrent_closes: usize,
250    /// Whether to enable dynamic size estimation for iceberg compaction.
251    #[serde(default = "default::storage::iceberg_compaction_enable_dynamic_size_estimation")]
252    pub iceberg_compaction_enable_dynamic_size_estimation: bool,
253    /// The smoothing factor for size estimation in iceberg compaction.(default: 0.3)
254    #[serde(default = "default::storage::iceberg_compaction_size_estimation_smoothing_factor")]
255    pub iceberg_compaction_size_estimation_smoothing_factor: f64,
256    /// Multiplier for pending waiting parallelism budget for iceberg compaction task queue.
257    /// Effective pending budget = `ceil(max_task_parallelism * multiplier)`. Default 4.0.
258    /// Set < 1.0 to reduce buffering (may increase `PullTask` RPC frequency); set higher to batch more tasks.
259    #[serde(
260        default = "default::storage::iceberg_compaction_pending_parallelism_budget_multiplier"
261    )]
262    pub iceberg_compaction_pending_parallelism_budget_multiplier: f32,
263    /// Maximum number of Iceberg compaction tasks requested in one pull.
264    #[serde(default = "default::storage::iceberg_compaction_max_pull_task_count")]
265    pub iceberg_compaction_max_pull_task_count: u32,
266    /// Pull interval for iceberg compaction task requests in milliseconds.
267    #[serde(
268        default = "default::storage::iceberg_compaction_pull_interval_ms",
269        deserialize_with = "deserialize_iceberg_compaction_pull_interval_ms"
270    )]
271    pub iceberg_compaction_pull_interval_ms: u64,
272    /// Enable prefetching entire data files before compacting them.
273    ///
274    /// When enabled, each input file is downloaded with a single HTTP GET before compaction
275    /// begins, replacing the default pattern of N+1 range reads (1 footer + N column chunks)
276    /// with a single sequential read per file. This reduces object storage READ API calls
277    /// from D×(1+N) to D per compaction cycle.
278    ///
279    /// Trade-off: higher peak memory — one full file is held in memory per concurrent
280    /// compaction task. Enable only when object storage API cost is a priority and memory headroom is
281    /// sufficient. See also the memory-protection config knobs such as
282    /// `iceberg_compaction_task_parallelism_ratio`.
283    #[serde(default = "default::storage::iceberg_compaction_enable_prefetch")]
284    pub iceberg_compaction_enable_prefetch: bool,
285
286    #[serde(default = "default::storage::iceberg_compaction_target_binpack_group_size_mb")]
287    pub iceberg_compaction_target_binpack_group_size_mb: Option<u64>,
288    #[serde(default = "default::storage::iceberg_compaction_min_group_size_mb")]
289    pub iceberg_compaction_min_group_size_mb: Option<u64>,
290    #[serde(default = "default::storage::iceberg_compaction_min_group_file_count")]
291    pub iceberg_compaction_min_group_file_count: Option<usize>,
292}
293
294/// the section `[storage.cache]` in `risingwave.toml`.
295#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
296#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
297pub struct CacheConfig {
298    /// Configure the capacity of the block cache in MB explicitly.
299    /// The overridden value will only be effective if:
300    /// 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
301    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
302    #[serde(default)]
303    pub block_cache_capacity_mb: Option<usize>,
304
305    /// Configure the number of shards in the block cache explicitly.
306    /// If not set, the shard number will be determined automatically based on cache capacity.
307    #[serde(default)]
308    pub block_cache_shard_num: Option<usize>,
309
310    #[serde(default)]
311    #[config_doc(omitted)]
312    pub block_cache_eviction: CacheEvictionConfig,
313
314    /// Configure the capacity of the block cache in MB explicitly.
315    /// The overridden value will only be effective if:
316    /// 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
317    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
318    #[serde(default)]
319    pub meta_cache_capacity_mb: Option<usize>,
320
321    /// Configure the number of shards in the meta cache explicitly.
322    /// If not set, the shard number will be determined automatically based on cache capacity.
323    #[serde(default)]
324    pub meta_cache_shard_num: Option<usize>,
325
326    #[serde(default)]
327    #[config_doc(omitted)]
328    pub meta_cache_eviction: CacheEvictionConfig,
329
330    #[serde(default = "default::storage::vector_block_cache_capacity_mb")]
331    pub vector_block_cache_capacity_mb: usize,
332    #[serde(default = "default::storage::vector_block_cache_shard_num")]
333    pub vector_block_cache_shard_num: usize,
334    #[serde(default)]
335    #[config_doc(omitted)]
336    pub vector_block_cache_eviction_config: CacheEvictionConfig,
337    #[serde(default = "default::storage::vector_meta_cache_capacity_mb")]
338    pub vector_meta_cache_capacity_mb: usize,
339    #[serde(default = "default::storage::vector_meta_cache_shard_num")]
340    pub vector_meta_cache_shard_num: usize,
341    #[serde(default)]
342    #[config_doc(omitted)]
343    pub vector_meta_cache_eviction_config: CacheEvictionConfig,
344}
345
346/// the section `[storage.cache.eviction]` in `risingwave.toml`.
347#[derive(Clone, Debug, Serialize, Deserialize)]
348#[serde(tag = "algorithm")]
349pub enum CacheEvictionConfig {
350    Lru {
351        high_priority_ratio_in_percent: Option<usize>,
352    },
353    Lfu {
354        window_capacity_ratio_in_percent: Option<usize>,
355        protected_capacity_ratio_in_percent: Option<usize>,
356        cmsketch_eps: Option<f64>,
357        cmsketch_confidence: Option<f64>,
358    },
359    S3Fifo {
360        small_queue_capacity_ratio_in_percent: Option<usize>,
361        ghost_queue_capacity_ratio_in_percent: Option<usize>,
362        small_to_main_freq_threshold: Option<u8>,
363    },
364}
365
366impl Default for CacheEvictionConfig {
367    fn default() -> Self {
368        Self::Lru {
369            high_priority_ratio_in_percent: None,
370        }
371    }
372}
373
374#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
375#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
376pub struct CacheRefillConfig {
377    /// Inflight meta cache refill tasks limit.
378    ///
379    /// 0 for unlimited.
380    #[serde(default = "default::cache_refill::meta_refill_concurrency")]
381    pub meta_refill_concurrency: usize,
382
383    /// `SSTable` levels to refill.
384    #[serde(default = "default::cache_refill::data_refill_levels")]
385    pub data_refill_levels: Vec<u32>,
386
387    /// Cache refill maximum timeout to apply version delta.
388    #[serde(default = "default::cache_refill::timeout_ms")]
389    pub timeout_ms: u64,
390
391    /// Inflight data cache refill tasks.
392    #[serde(default = "default::cache_refill::concurrency")]
393    pub concurrency: usize,
394
395    /// Block count that a data cache refill request fetches.
396    #[serde(default = "default::cache_refill::unit")]
397    pub unit: usize,
398
399    /// Data cache refill unit admission ratio.
400    ///
401    /// Only unit whose blocks are admitted above the ratio will be refilled.
402    #[serde(default = "default::cache_refill::threshold")]
403    pub threshold: f64,
404
405    /// Recent filter layer shards.
406    #[serde(default = "default::cache_refill::recent_filter_shards")]
407    pub recent_filter_shards: usize,
408
409    /// Recent filter layer count.
410    #[serde(default = "default::cache_refill::recent_filter_layers")]
411    pub recent_filter_layers: usize,
412
413    /// Recent filter layer rotate interval.
414    #[serde(default = "default::cache_refill::recent_filter_rotate_interval_ms")]
415    pub recent_filter_rotate_interval_ms: usize,
416
417    /// Skip checking recent filter on data refill.
418    ///
419    /// This option is suitable for a single compute node or debugging.
420    #[serde(default = "default::cache_refill::skip_recent_filter")]
421    pub skip_recent_filter: bool,
422
423    /// Skip checking inheritance filter on data refill.
424    ///
425    /// The inheritance filter only runs after recent-filter admission, so this
426    /// option has no effect when `skip_recent_filter` is enabled.
427    ///
428    /// This option is suitable for a single compute node or debugging.
429    #[serde(default = "default::cache_refill::skip_inheritance_filter")]
430    pub skip_inheritance_filter: bool,
431
432    #[serde(default, flatten)]
433    #[config_doc(omitted)]
434    pub unrecognized: Unrecognized<Self>,
435}
436
437#[derive(Clone, Debug, Default, Serialize, Deserialize)]
438pub struct FileCacheTokioRuntimeConfig {
439    /// Dedicated runtime worker threads. `0` uses the Tokio default.
440    pub worker_threads: usize,
441
442    /// Maximum number of blocking threads. `0` uses the Tokio default.
443    pub max_blocking_threads: usize,
444}
445
446#[derive(Clone, Debug, Serialize, Deserialize)]
447pub enum FileCacheRuntimeConfig {
448    /// Use the runtime that creates the file cache.
449    Disabled,
450    /// Use one dedicated runtime for all file cache tasks.
451    Unified(FileCacheTokioRuntimeConfig),
452    /// Legacy configuration for separate read and write runtimes.
453    ///
454    /// Foyer 0.22 uses one spawner, so this configuration is no longer supported.
455    Separated {
456        read_runtime_options: FileCacheTokioRuntimeConfig,
457        write_runtime_options: FileCacheTokioRuntimeConfig,
458    },
459}
460
461/// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`.
462///
463/// It's put at [`StorageConfig::data_file_cache`] and  [`StorageConfig::meta_file_cache`].
464#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
465#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
466pub struct FileCacheConfig {
467    #[serde(default = "default::file_cache::dir")]
468    pub dir: String,
469
470    /// Whether to use direct I/O for file cache reads and writes on Linux.
471    #[serde(default = "default::file_cache::direct_io")]
472    pub direct_io: bool,
473
474    #[serde(default = "default::file_cache::capacity_mb")]
475    pub capacity_mb: usize,
476
477    #[serde(default = "default::file_cache::file_capacity_mb")]
478    pub file_capacity_mb: usize,
479
480    #[serde(default = "default::file_cache::flushers")]
481    pub flushers: usize,
482
483    #[serde(default = "default::file_cache::reclaimers")]
484    pub reclaimers: usize,
485
486    #[serde(default = "default::file_cache::recover_concurrency")]
487    pub recover_concurrency: usize,
488
489    /// Deprecated soon. Please use `throttle` to do I/O throttling instead.
490    #[serde(default = "default::file_cache::insert_rate_limit_mb")]
491    pub insert_rate_limit_mb: usize,
492
493    #[serde(default = "default::file_cache::indexer_shards")]
494    pub indexer_shards: usize,
495
496    #[serde(default = "default::file_cache::compression")]
497    pub compression: Compression,
498
499    #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
500    pub flush_buffer_threshold_mb: Option<usize>,
501
502    /// Maximum estimated size of cache entries waiting in the flusher submit queues. New cache
503    /// entries are ignored when the pending size exceeds this threshold.
504    #[serde(default = "default::file_cache::submit_queue_size_threshold_mb")]
505    pub submit_queue_size_threshold_mb: usize,
506
507    #[serde(default = "default::file_cache::throttle")]
508    pub throttle: Throttle,
509
510    #[serde(default = "default::file_cache::fifo_probation_ratio")]
511    pub fifo_probation_ratio: f64,
512
513    /// Set the blob index size for each blob.
514    ///
515    /// A larger blob index size can hold more blob entries, but it will also increase the io size of each blob part
516    /// write.
517    ///
518    /// NOTE:
519    ///
520    /// - The size will be aligned up to a multiplier of 4K.
521    /// - Modifying this configuration will invalidate all existing file cache data.
522    ///
523    /// Default: 16 `KiB`
524    #[serde(default = "default::file_cache::blob_index_size_kb")]
525    pub blob_index_size_kb: usize,
526
527    /// Recover mode.
528    ///
529    /// Options:
530    ///
531    /// - "None": Do not recover disk cache.
532    /// - "Quiet": Recover disk cache and skip errors.
533    /// - "Strict": Recover disk cache and panic on errors.
534    ///
535    /// More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`],
536    #[serde(default = "default::file_cache::recover_mode")]
537    pub recover_mode: RecoverMode,
538
539    #[serde(default = "default::file_cache::runtime_config")]
540    pub runtime_config: FileCacheRuntimeConfig,
541
542    #[serde(default, flatten)]
543    #[config_doc(omitted)]
544    pub unrecognized: Unrecognized<Self>,
545}
546
547/// The subsections `[storage.object_store]`.
548#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
549#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
550pub struct ObjectStoreConfig {
551    // alias is for backward compatibility
552    #[serde(
553        default = "default::object_store_config::set_atomic_write_dir",
554        alias = "object_store_set_atomic_write_dir"
555    )]
556    pub set_atomic_write_dir: bool,
557
558    /// Retry and timeout configuration
559    /// Description retry strategy driven by exponential back-off
560    /// Exposes the timeout and retries of each Object store interface. Therefore, the total timeout for each interface is determined based on the interface's timeout/retry configuration and the exponential back-off policy.
561    #[serde(default)]
562    pub retry: ObjectStoreRetryConfig,
563
564    /// Some special configuration of S3 Backend
565    #[serde(default)]
566    pub s3: S3ObjectStoreConfig,
567
568    /// Maximum number of concurrent object store requests (read, `streaming_read`, metadata, etc.).
569    /// 0 means unlimited. When set to a positive value, a semaphore will be used to limit
570    /// the number of in-flight requests to the object store, preventing HTTP connection pool
571    /// contention under high concurrency.
572    #[serde(default = "default::object_store_config::object_store_req_concurrency_limit")]
573    pub req_concurrency_limit: usize,
574
575    /// Maximum number of concurrent HTTP requests used by the `OpenDAL` GCS backend.
576    /// 0 means unlimited.
577    #[serde(default = "default::object_store_config::http_concurrent_limit")]
578    pub http_concurrent_limit: usize,
579
580    // TODO: the following field will be deprecated after opendal is stabilized
581    #[serde(default = "default::object_store_config::opendal_upload_concurrency")]
582    pub opendal_upload_concurrency: usize,
583
584    // TODO: the following field will be deprecated after opendal is stabilized
585    #[serde(default)]
586    pub opendal_writer_abort_on_err: bool,
587
588    #[serde(default = "default::object_store_config::upload_part_size")]
589    pub upload_part_size: usize,
590}
591
592fn deserialize_max_prefetch_block_number<'de, D>(deserializer: D) -> Result<usize, D::Error>
593where
594    D: serde::Deserializer<'de>,
595{
596    let value = usize::deserialize(deserializer)?;
597    if value == 0 {
598        return Err(D::Error::custom(
599            "storage.max_prefetch_block_number must be greater than 0",
600        ));
601    }
602    Ok(value)
603}
604
605fn deserialize_iceberg_compaction_pull_interval_ms<'de, D>(deserializer: D) -> Result<u64, D::Error>
606where
607    D: serde::Deserializer<'de>,
608{
609    let value = u64::deserialize(deserializer)?;
610    if value == 0 {
611        return Err(D::Error::custom(
612            "storage.iceberg_compaction_pull_interval_ms must be greater than 0",
613        ));
614    }
615    Ok(value)
616}
617
618impl ObjectStoreConfig {
619    pub fn set_atomic_write_dir(&mut self) {
620        self.set_atomic_write_dir = true;
621    }
622}
623
624/// The subsections `[storage.object_store.s3]`.
625#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
626#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
627pub struct S3ObjectStoreConfig {
628    // alias is for backward compatibility
629    #[serde(
630        default = "default::object_store_config::s3::keepalive_ms",
631        alias = "object_store_keepalive_ms"
632    )]
633    pub keepalive_ms: Option<u64>,
634    #[serde(
635        default = "default::object_store_config::s3::recv_buffer_size",
636        alias = "object_store_recv_buffer_size"
637    )]
638    pub recv_buffer_size: Option<usize>,
639    #[serde(
640        default = "default::object_store_config::s3::send_buffer_size",
641        alias = "object_store_send_buffer_size"
642    )]
643    pub send_buffer_size: Option<usize>,
644    #[serde(
645        default = "default::object_store_config::s3::nodelay",
646        alias = "object_store_nodelay"
647    )]
648    pub nodelay: Option<bool>,
649    /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead.
650    #[serde(default = "default::object_store_config::s3::developer::retry_unknown_service_error")]
651    pub retry_unknown_service_error: bool,
652    #[serde(default = "default::object_store_config::s3::identity_resolution_timeout_s")]
653    pub identity_resolution_timeout_s: u64,
654    #[serde(default)]
655    pub developer: S3ObjectStoreDeveloperConfig,
656}
657
658/// The subsections `[storage.object_store.s3.developer]`.
659#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
660#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
661pub struct S3ObjectStoreDeveloperConfig {
662    /// Whether to retry s3 sdk error from which no error metadata is provided.
663    #[serde(
664        default = "default::object_store_config::s3::developer::retry_unknown_service_error",
665        alias = "object_store_retry_unknown_service_error"
666    )]
667    pub retry_unknown_service_error: bool,
668    /// An array of error codes that should be retried.
669    /// e.g. `["SlowDown", "TooManyRequests"]`
670    #[serde(
671        default = "default::object_store_config::s3::developer::retryable_service_error_codes",
672        alias = "object_store_retryable_service_error_codes"
673    )]
674    pub retryable_service_error_codes: Vec<String>,
675
676    // TODO: deprecate this config when we are completely deprecate aws sdk.
677    #[serde(default = "default::object_store_config::s3::developer::use_opendal")]
678    pub use_opendal: bool,
679}
680
681#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
682#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
683pub struct ObjectStoreRetryConfig {
684    // A retry strategy driven by exponential back-off.
685    // The retry strategy is used for all object store operations.
686    /// Given a base duration for retry strategy in milliseconds.
687    #[serde(default = "default::object_store_config::object_store_req_backoff_interval_ms")]
688    pub req_backoff_interval_ms: u64,
689
690    /// The max delay interval for the retry strategy. No retry delay will be longer than this `Duration`.
691    #[serde(default = "default::object_store_config::object_store_req_backoff_max_delay_ms")]
692    pub req_backoff_max_delay_ms: u64,
693
694    /// A multiplicative factor that will be applied to the exponential back-off retry delay.
695    #[serde(default = "default::object_store_config::object_store_req_backoff_factor")]
696    pub req_backoff_factor: u64,
697
698    /// Maximum timeout for `upload` operation
699    #[serde(default = "default::object_store_config::object_store_upload_attempt_timeout_ms")]
700    pub upload_attempt_timeout_ms: u64,
701
702    /// Total counts of `upload` operation retries
703    #[serde(default = "default::object_store_config::object_store_upload_retry_attempts")]
704    pub upload_retry_attempts: usize,
705
706    /// Maximum timeout for `streaming_upload_init` and `streaming_upload`
707    #[serde(
708        default = "default::object_store_config::object_store_streaming_upload_attempt_timeout_ms"
709    )]
710    pub streaming_upload_attempt_timeout_ms: u64,
711
712    /// Total counts of `streaming_upload` operation retries
713    #[serde(
714        default = "default::object_store_config::object_store_streaming_upload_retry_attempts"
715    )]
716    pub streaming_upload_retry_attempts: usize,
717
718    /// Maximum timeout for `read` operation
719    #[serde(default = "default::object_store_config::object_store_read_attempt_timeout_ms")]
720    pub read_attempt_timeout_ms: u64,
721
722    /// Total counts of `read` operation retries
723    #[serde(default = "default::object_store_config::object_store_read_retry_attempts")]
724    pub read_retry_attempts: usize,
725
726    /// Maximum timeout for `streaming_read_init` and `streaming_read` operation
727    #[serde(
728        default = "default::object_store_config::object_store_streaming_read_attempt_timeout_ms"
729    )]
730    pub streaming_read_attempt_timeout_ms: u64,
731
732    /// Total counts of `streaming_read operation` retries
733    #[serde(default = "default::object_store_config::object_store_streaming_read_retry_attempts")]
734    pub streaming_read_retry_attempts: usize,
735
736    /// Maximum timeout for `metadata` operation
737    #[serde(default = "default::object_store_config::object_store_metadata_attempt_timeout_ms")]
738    pub metadata_attempt_timeout_ms: u64,
739
740    /// Total counts of `metadata` operation retries
741    #[serde(default = "default::object_store_config::object_store_metadata_retry_attempts")]
742    pub metadata_retry_attempts: usize,
743
744    /// Maximum timeout for `delete` operation
745    #[serde(default = "default::object_store_config::object_store_delete_attempt_timeout_ms")]
746    pub delete_attempt_timeout_ms: u64,
747
748    /// Total counts of `delete` operation retries
749    #[serde(default = "default::object_store_config::object_store_delete_retry_attempts")]
750    pub delete_retry_attempts: usize,
751
752    /// Maximum timeout for `delete_object` operation
753    #[serde(
754        default = "default::object_store_config::object_store_delete_objects_attempt_timeout_ms"
755    )]
756    pub delete_objects_attempt_timeout_ms: u64,
757
758    /// Total counts of `delete_object` operation retries
759    #[serde(default = "default::object_store_config::object_store_delete_objects_retry_attempts")]
760    pub delete_objects_retry_attempts: usize,
761
762    /// Maximum timeout for `list` operation
763    #[serde(default = "default::object_store_config::object_store_list_attempt_timeout_ms")]
764    pub list_attempt_timeout_ms: u64,
765
766    /// Total counts of `list` operation retries
767    #[serde(default = "default::object_store_config::object_store_list_retry_attempts")]
768    pub list_retry_attempts: usize,
769}
770
771#[derive(Debug, Clone)]
772pub enum EvictionConfig {
773    Lru(LruConfig),
774    Lfu(LfuConfig),
775    S3Fifo(S3FifoConfig),
776}
777
778impl EvictionConfig {
779    pub fn for_test() -> Self {
780        Self::Lru(LruConfig {
781            high_priority_pool_ratio: 0.0,
782        })
783    }
784}
785
786impl From<EvictionConfig> for foyer::EvictionConfig {
787    fn from(value: EvictionConfig) -> Self {
788        match value {
789            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
790            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
791            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
792        }
793    }
794}
795
796pub struct StorageMemoryConfig {
797    pub block_cache_capacity_mb: usize,
798    pub block_cache_shard_num: usize,
799    pub meta_cache_capacity_mb: usize,
800    pub meta_cache_shard_num: usize,
801    pub vector_block_cache_capacity_mb: usize,
802    pub vector_block_cache_shard_num: usize,
803    pub vector_meta_cache_capacity_mb: usize,
804    pub vector_meta_cache_shard_num: usize,
805    pub shared_buffer_capacity_mb: usize,
806    pub compactor_memory_limit_mb: usize,
807    pub prefetch_buffer_capacity_mb: usize,
808    pub block_cache_eviction_config: EvictionConfig,
809    pub meta_cache_eviction_config: EvictionConfig,
810    pub vector_block_cache_eviction_config: EvictionConfig,
811    pub vector_meta_cache_eviction_config: EvictionConfig,
812    pub block_file_cache_flush_buffer_threshold_mb: usize,
813    pub meta_file_cache_flush_buffer_threshold_mb: usize,
814}
815
816pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
817    let block_cache_capacity_mb = s.storage.cache.block_cache_capacity_mb.unwrap_or(
818        // adapt to old version
819        s.storage
820            .block_cache_capacity_mb
821            .unwrap_or(default::storage::block_cache_capacity_mb()),
822    );
823    let meta_cache_capacity_mb = s.storage.cache.meta_cache_capacity_mb.unwrap_or(
824        // adapt to old version
825        s.storage
826            .block_cache_capacity_mb
827            .unwrap_or(default::storage::meta_cache_capacity_mb()),
828    );
829    let shared_buffer_capacity_mb = s
830        .storage
831        .shared_buffer_capacity_mb
832        .unwrap_or(default::storage::shared_buffer_capacity_mb());
833    let meta_cache_shard_num = s.storage.cache.meta_cache_shard_num.unwrap_or_else(|| {
834        let mut shard_bits = MAX_META_CACHE_SHARD_BITS;
835        while (meta_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0 {
836            shard_bits -= 1;
837        }
838        shard_bits
839    });
840    let block_cache_shard_num = s.storage.cache.block_cache_shard_num.unwrap_or_else(|| {
841        let mut shard_bits = MAX_BLOCK_CACHE_SHARD_BITS;
842        while (block_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0
843        {
844            shard_bits -= 1;
845        }
846        shard_bits
847    });
848    let compactor_memory_limit_mb = s
849        .storage
850        .compactor_memory_limit_mb
851        .unwrap_or(default::storage::compactor_memory_limit_mb());
852
853    let get_eviction_config = |c: &CacheEvictionConfig| {
854        match c {
855            CacheEvictionConfig::Lru {
856                high_priority_ratio_in_percent,
857            } => EvictionConfig::Lru(LruConfig {
858                high_priority_pool_ratio: high_priority_ratio_in_percent.unwrap_or(
859                    // adapt to old version
860                    s.storage
861                        .high_priority_ratio_in_percent
862                        .unwrap_or(default::storage::high_priority_ratio_in_percent()),
863                ) as f64
864                    / 100.0,
865            }),
866            CacheEvictionConfig::Lfu {
867                window_capacity_ratio_in_percent,
868                protected_capacity_ratio_in_percent,
869                cmsketch_eps,
870                cmsketch_confidence,
871            } => EvictionConfig::Lfu(LfuConfig {
872                window_capacity_ratio: window_capacity_ratio_in_percent
873                    .unwrap_or(default::storage::window_capacity_ratio_in_percent())
874                    as f64
875                    / 100.0,
876                protected_capacity_ratio: protected_capacity_ratio_in_percent
877                    .unwrap_or(default::storage::protected_capacity_ratio_in_percent())
878                    as f64
879                    / 100.0,
880                cmsketch_eps: cmsketch_eps.unwrap_or(default::storage::cmsketch_eps()),
881                cmsketch_confidence: cmsketch_confidence
882                    .unwrap_or(default::storage::cmsketch_confidence()),
883            }),
884            CacheEvictionConfig::S3Fifo {
885                small_queue_capacity_ratio_in_percent,
886                ghost_queue_capacity_ratio_in_percent,
887                small_to_main_freq_threshold,
888            } => EvictionConfig::S3Fifo(S3FifoConfig {
889                small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
890                    .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
891                    as f64
892                    / 100.0,
893                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
894                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
895                    as f64
896                    / 100.0,
897                small_to_main_freq_threshold: small_to_main_freq_threshold
898                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
899            }),
900        }
901    };
902
903    let block_cache_eviction_config = get_eviction_config(&s.storage.cache.block_cache_eviction);
904    let meta_cache_eviction_config = get_eviction_config(&s.storage.cache.meta_cache_eviction);
905    let vector_block_cache_eviction_config =
906        get_eviction_config(&s.storage.cache.vector_block_cache_eviction_config);
907    let vector_meta_cache_eviction_config =
908        get_eviction_config(&s.storage.cache.vector_meta_cache_eviction_config);
909
910    let prefetch_buffer_capacity_mb =
911        s.storage
912            .shared_buffer_capacity_mb
913            .unwrap_or(match &block_cache_eviction_config {
914                EvictionConfig::Lru(lru) => {
915                    ((1.0 - lru.high_priority_pool_ratio) * block_cache_capacity_mb as f64) as usize
916                }
917                EvictionConfig::Lfu(lfu) => {
918                    ((1.0 - lfu.protected_capacity_ratio) * block_cache_capacity_mb as f64) as usize
919                }
920                EvictionConfig::S3Fifo(s3fifo) => {
921                    (s3fifo.small_queue_capacity_ratio * block_cache_capacity_mb as f64) as usize
922                }
923            });
924
925    let block_file_cache_flush_buffer_threshold_mb = s
926        .storage
927        .data_file_cache
928        .flush_buffer_threshold_mb
929        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
930    let meta_file_cache_flush_buffer_threshold_mb = s
931        .storage
932        .meta_file_cache
933        .flush_buffer_threshold_mb
934        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
935
936    StorageMemoryConfig {
937        block_cache_capacity_mb,
938        block_cache_shard_num,
939        meta_cache_capacity_mb,
940        meta_cache_shard_num,
941        vector_block_cache_capacity_mb: s.storage.cache.vector_block_cache_capacity_mb,
942        vector_block_cache_shard_num: s.storage.cache.vector_block_cache_shard_num,
943        vector_meta_cache_capacity_mb: s.storage.cache.vector_meta_cache_capacity_mb,
944        vector_meta_cache_shard_num: s.storage.cache.vector_meta_cache_shard_num,
945        shared_buffer_capacity_mb,
946        compactor_memory_limit_mb,
947        prefetch_buffer_capacity_mb,
948        block_cache_eviction_config,
949        meta_cache_eviction_config,
950        vector_block_cache_eviction_config,
951        vector_meta_cache_eviction_config,
952        block_file_cache_flush_buffer_threshold_mb,
953        meta_file_cache_flush_buffer_threshold_mb,
954    }
955}
956
957pub mod default {
958
959    pub mod storage {
960        pub fn share_buffers_sync_parallelism() -> u32 {
961            1
962        }
963
964        pub fn share_buffer_compaction_worker_threads_number() -> u32 {
965            4
966        }
967
968        pub fn shared_buffer_capacity_mb() -> usize {
969            1024
970        }
971
972        pub fn shared_buffer_flush_ratio() -> f32 {
973            0.8
974        }
975
976        pub fn shared_buffer_min_batch_flush_size_mb() -> usize {
977            800
978        }
979
980        pub fn imm_merge_threshold() -> usize {
981            0 // disable
982        }
983
984        pub fn write_conflict_detection_enabled() -> bool {
985            cfg!(debug_assertions)
986        }
987
988        pub fn max_cached_recent_versions_number() -> usize {
989            60
990        }
991
992        pub fn block_cache_capacity_mb() -> usize {
993            512
994        }
995
996        pub fn high_priority_ratio_in_percent() -> usize {
997            70
998        }
999
1000        pub fn window_capacity_ratio_in_percent() -> usize {
1001            10
1002        }
1003
1004        pub fn protected_capacity_ratio_in_percent() -> usize {
1005            80
1006        }
1007
1008        pub fn cmsketch_eps() -> f64 {
1009            0.002
1010        }
1011
1012        pub fn cmsketch_confidence() -> f64 {
1013            0.95
1014        }
1015
1016        pub fn small_queue_capacity_ratio_in_percent() -> usize {
1017            10
1018        }
1019
1020        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
1021            1000
1022        }
1023
1024        pub fn small_to_main_freq_threshold() -> u8 {
1025            1
1026        }
1027
1028        pub fn meta_cache_capacity_mb() -> usize {
1029            128
1030        }
1031
1032        pub fn disable_remote_compactor() -> bool {
1033            false
1034        }
1035
1036        pub fn share_buffer_upload_concurrency() -> usize {
1037            8
1038        }
1039
1040        pub fn compactor_memory_limit_mb() -> usize {
1041            512
1042        }
1043
1044        pub fn compactor_max_task_multiplier() -> f32 {
1045            3.0
1046        }
1047
1048        pub fn compactor_memory_available_proportion() -> f64 {
1049            0.8
1050        }
1051
1052        pub fn sstable_id_remote_fetch_number() -> u32 {
1053            10
1054        }
1055
1056        pub fn min_sstable_size_mb() -> u32 {
1057            32
1058        }
1059
1060        pub fn min_sst_size_for_streaming_upload() -> u64 {
1061            // 32MB
1062            32 * 1024 * 1024
1063        }
1064
1065        pub fn max_concurrent_compaction_task_number() -> u64 {
1066            16
1067        }
1068
1069        pub fn max_preload_wait_time_mill() -> u64 {
1070            0
1071        }
1072
1073        pub fn max_version_pinning_duration_sec() -> u64 {
1074            3 * 3600
1075        }
1076
1077        pub fn compactor_max_sst_key_count() -> u64 {
1078            2 * 1024 * 1024 // 200w
1079        }
1080
1081        pub fn compact_iter_recreate_timeout_ms() -> u64 {
1082            10 * 60 * 1000
1083        }
1084
1085        pub fn compactor_iter_max_io_retry_times() -> usize {
1086            8
1087        }
1088
1089        pub fn shorten_block_meta_key_threshold() -> Option<usize> {
1090            None
1091        }
1092
1093        pub fn compactor_max_sst_size() -> u64 {
1094            512 * 1024 * 1024 // 512m
1095        }
1096
1097        pub fn enable_fast_compaction() -> bool {
1098            true
1099        }
1100
1101        pub fn check_compaction_result() -> bool {
1102            false
1103        }
1104
1105        pub fn max_preload_io_retry_times() -> usize {
1106            3
1107        }
1108
1109        pub fn mem_table_spill_threshold() -> usize {
1110            4 << 20
1111        }
1112
1113        pub fn compactor_fast_max_compact_delete_ratio() -> u32 {
1114            40
1115        }
1116
1117        pub fn compactor_fast_max_compact_task_size() -> u64 {
1118            2 * 1024 * 1024 * 1024 // 2g
1119        }
1120
1121        pub fn max_prefetch_block_number() -> usize {
1122            16
1123        }
1124
1125        pub fn compactor_concurrent_uploading_sst_count() -> Option<usize> {
1126            None
1127        }
1128
1129        pub fn compactor_max_overlap_sst_count() -> usize {
1130            64
1131        }
1132
1133        pub fn compactor_max_preload_meta_file_count() -> usize {
1134            32
1135        }
1136
1137        pub fn vector_file_block_size_kb() -> usize {
1138            1024
1139        }
1140
1141        pub fn vector_block_cache_capacity_mb() -> usize {
1142            16
1143        }
1144
1145        pub fn vector_block_cache_shard_num() -> usize {
1146            16
1147        }
1148
1149        pub fn vector_meta_cache_capacity_mb() -> usize {
1150            16
1151        }
1152
1153        pub fn vector_meta_cache_shard_num() -> usize {
1154            16
1155        }
1156
1157        // deprecated
1158        pub fn table_info_statistic_history_times() -> usize {
1159            240
1160        }
1161
1162        pub fn block_file_cache_flush_buffer_threshold_mb() -> usize {
1163            256
1164        }
1165
1166        pub fn meta_file_cache_flush_buffer_threshold_mb() -> usize {
1167            64
1168        }
1169
1170        pub fn time_travel_version_cache_capacity() -> u64 {
1171            10
1172        }
1173
1174        pub fn table_change_log_cache_capacity() -> u64 {
1175            60
1176        }
1177
1178        pub fn sst_skip_bloom_filter_in_serde() -> bool {
1179            false
1180        }
1181
1182        pub fn iceberg_compaction_enable_validate() -> bool {
1183            false
1184        }
1185
1186        pub fn iceberg_compaction_max_record_batch_rows() -> usize {
1187            1024
1188        }
1189
1190        pub fn iceberg_compaction_write_parquet_max_row_group_rows() -> usize {
1191            1024 * 100 // 100k
1192        }
1193
1194        pub fn iceberg_compaction_min_size_per_partition_mb() -> u32 {
1195            1024
1196        }
1197
1198        pub fn iceberg_compaction_max_file_count_per_partition() -> u32 {
1199            32
1200        }
1201
1202        pub fn iceberg_compaction_task_parallelism_ratio() -> f32 {
1203            4.0
1204        }
1205
1206        pub fn iceberg_compaction_enable_heuristic_output_parallelism() -> bool {
1207            false
1208        }
1209
1210        pub fn iceberg_compaction_max_concurrent_closes() -> usize {
1211            8
1212        }
1213
1214        pub fn iceberg_compaction_enable_dynamic_size_estimation() -> bool {
1215            true
1216        }
1217
1218        pub fn iceberg_compaction_size_estimation_smoothing_factor() -> f64 {
1219            0.3
1220        }
1221
1222        pub fn iceberg_compaction_pending_parallelism_budget_multiplier() -> f32 {
1223            4.0
1224        }
1225
1226        pub fn iceberg_compaction_max_pull_task_count() -> u32 {
1227            1
1228        }
1229
1230        pub fn iceberg_compaction_pull_interval_ms() -> u64 {
1231            5000
1232        }
1233
1234        pub fn iceberg_compaction_enable_prefetch() -> bool {
1235            false
1236        }
1237
1238        pub fn iceberg_compaction_target_binpack_group_size_mb() -> Option<u64> {
1239            Some(100 * 1024) // 100GB
1240        }
1241
1242        pub fn iceberg_compaction_min_group_size_mb() -> Option<u64> {
1243            None
1244        }
1245
1246        pub fn iceberg_compaction_min_group_file_count() -> Option<usize> {
1247            None
1248        }
1249    }
1250
1251    pub mod file_cache {
1252        use std::num::NonZeroUsize;
1253
1254        use foyer::{Compression, RecoverMode, Throttle};
1255
1256        use super::super::{FileCacheRuntimeConfig, FileCacheTokioRuntimeConfig};
1257
1258        pub fn dir() -> String {
1259            "".to_owned()
1260        }
1261
1262        pub fn direct_io() -> bool {
1263            false
1264        }
1265
1266        pub fn capacity_mb() -> usize {
1267            1024
1268        }
1269
1270        pub fn file_capacity_mb() -> usize {
1271            64
1272        }
1273
1274        pub fn flushers() -> usize {
1275            4
1276        }
1277
1278        pub fn reclaimers() -> usize {
1279            4
1280        }
1281
1282        pub fn recover_concurrency() -> usize {
1283            8
1284        }
1285
1286        pub fn insert_rate_limit_mb() -> usize {
1287            0
1288        }
1289
1290        pub fn indexer_shards() -> usize {
1291            64
1292        }
1293
1294        pub fn compression() -> Compression {
1295            Compression::None
1296        }
1297
1298        pub fn flush_buffer_threshold_mb() -> Option<usize> {
1299            None
1300        }
1301
1302        pub fn submit_queue_size_threshold_mb() -> usize {
1303            16
1304        }
1305
1306        pub fn fifo_probation_ratio() -> f64 {
1307            0.1
1308        }
1309
1310        pub fn blob_index_size_kb() -> usize {
1311            16
1312        }
1313
1314        pub fn recover_mode() -> RecoverMode {
1315            RecoverMode::Quiet
1316        }
1317
1318        pub fn runtime_config() -> FileCacheRuntimeConfig {
1319            FileCacheRuntimeConfig::Unified(FileCacheTokioRuntimeConfig::default())
1320        }
1321
1322        pub fn throttle() -> Throttle {
1323            Throttle::new()
1324                .with_iops_counter(foyer::IopsCounter::PerIoSize(
1325                    NonZeroUsize::new(128 * 1024).unwrap(),
1326                ))
1327                .with_read_iops(100000)
1328                .with_write_iops(100000)
1329                .with_write_throughput(1024 * 1024 * 1024)
1330                .with_read_throughput(1024 * 1024 * 1024)
1331        }
1332    }
1333
1334    pub mod cache_refill {
1335        pub fn meta_refill_concurrency() -> usize {
1336            0
1337        }
1338
1339        pub fn data_refill_levels() -> Vec<u32> {
1340            vec![]
1341        }
1342
1343        pub fn timeout_ms() -> u64 {
1344            6000
1345        }
1346
1347        pub fn concurrency() -> usize {
1348            10
1349        }
1350
1351        pub fn unit() -> usize {
1352            64
1353        }
1354
1355        pub fn threshold() -> f64 {
1356            0.5
1357        }
1358
1359        pub fn recent_filter_shards() -> usize {
1360            16
1361        }
1362
1363        pub fn recent_filter_layers() -> usize {
1364            6
1365        }
1366
1367        pub fn recent_filter_rotate_interval_ms() -> usize {
1368            10000
1369        }
1370
1371        pub fn skip_recent_filter() -> bool {
1372            false
1373        }
1374
1375        pub fn skip_inheritance_filter() -> bool {
1376            false
1377        }
1378    }
1379
1380    pub mod object_store_config {
1381        const DEFAULT_REQ_BACKOFF_INTERVAL_MS: u64 = 1000; // 1s
1382        const DEFAULT_REQ_BACKOFF_MAX_DELAY_MS: u64 = 10 * 1000; // 10s
1383        const DEFAULT_REQ_MAX_RETRY_ATTEMPTS: usize = 3;
1384
1385        pub fn set_atomic_write_dir() -> bool {
1386            false
1387        }
1388
1389        pub fn object_store_req_concurrency_limit() -> usize {
1390            0
1391        }
1392
1393        pub fn http_concurrent_limit() -> usize {
1394            0
1395        }
1396
1397        pub fn object_store_req_backoff_interval_ms() -> u64 {
1398            DEFAULT_REQ_BACKOFF_INTERVAL_MS
1399        }
1400
1401        pub fn object_store_req_backoff_max_delay_ms() -> u64 {
1402            DEFAULT_REQ_BACKOFF_MAX_DELAY_MS // 10s
1403        }
1404
1405        pub fn object_store_req_backoff_factor() -> u64 {
1406            2
1407        }
1408
1409        pub fn object_store_upload_attempt_timeout_ms() -> u64 {
1410            8 * 1000 // 8s
1411        }
1412
1413        pub fn object_store_upload_retry_attempts() -> usize {
1414            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1415        }
1416
1417        // init + upload_part + finish
1418        pub fn object_store_streaming_upload_attempt_timeout_ms() -> u64 {
1419            5 * 1000 // 5s
1420        }
1421
1422        pub fn object_store_streaming_upload_retry_attempts() -> usize {
1423            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1424        }
1425
1426        // tips: depend on block_size
1427        pub fn object_store_read_attempt_timeout_ms() -> u64 {
1428            8 * 1000 // 8s
1429        }
1430
1431        pub fn object_store_read_retry_attempts() -> usize {
1432            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1433        }
1434
1435        pub fn object_store_streaming_read_attempt_timeout_ms() -> u64 {
1436            3 * 1000 // 3s
1437        }
1438
1439        pub fn object_store_streaming_read_retry_attempts() -> usize {
1440            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1441        }
1442
1443        pub fn object_store_metadata_attempt_timeout_ms() -> u64 {
1444            60 * 1000 // 1min
1445        }
1446
1447        pub fn object_store_metadata_retry_attempts() -> usize {
1448            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1449        }
1450
1451        pub fn object_store_delete_attempt_timeout_ms() -> u64 {
1452            5 * 1000
1453        }
1454
1455        pub fn object_store_delete_retry_attempts() -> usize {
1456            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1457        }
1458
1459        // tips: depend on batch size
1460        pub fn object_store_delete_objects_attempt_timeout_ms() -> u64 {
1461            5 * 1000
1462        }
1463
1464        pub fn object_store_delete_objects_retry_attempts() -> usize {
1465            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1466        }
1467
1468        pub fn object_store_list_attempt_timeout_ms() -> u64 {
1469            10 * 60 * 1000
1470        }
1471
1472        pub fn object_store_list_retry_attempts() -> usize {
1473            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1474        }
1475
1476        pub fn opendal_upload_concurrency() -> usize {
1477            256
1478        }
1479
1480        pub fn upload_part_size() -> usize {
1481            // 16m
1482            16 * 1024 * 1024
1483        }
1484
1485        pub mod s3 {
1486            const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
1487
1488            const DEFAULT_KEEPALIVE_MS: u64 = 600 * 1000; // 10min
1489
1490            pub fn keepalive_ms() -> Option<u64> {
1491                Some(DEFAULT_KEEPALIVE_MS) // 10min
1492            }
1493
1494            pub fn recv_buffer_size() -> Option<usize> {
1495                Some(1 << 21) // 2m
1496            }
1497
1498            pub fn send_buffer_size() -> Option<usize> {
1499                None
1500            }
1501
1502            pub fn nodelay() -> Option<bool> {
1503                Some(true)
1504            }
1505
1506            pub fn identity_resolution_timeout_s() -> u64 {
1507                DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S
1508            }
1509
1510            pub mod developer {
1511                pub fn retry_unknown_service_error() -> bool {
1512                    false
1513                }
1514
1515                pub fn retryable_service_error_codes() -> Vec<String> {
1516                    vec!["SlowDown".into(), "TooManyRequests".into()]
1517                }
1518
1519                pub fn use_opendal() -> bool {
1520                    true
1521                }
1522            }
1523        }
1524    }
1525}
risingwave_common/config/storage.rs

risingwave_common/config/
storage.rs
