risingwave_common/config/
storage.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use foyer::{Compression, LfuConfig, LruConfig, RecoverMode, S3FifoConfig, Throttle};
16use serde::de::Error as _;
17
18use super::*;
19
20/// The section `[storage]` in `risingwave.toml`.
21#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
22#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
23pub struct StorageConfig {
24    /// parallelism while syncing share buffers into L0 SST. Should NOT be 0.
25    #[serde(default = "default::storage::share_buffers_sync_parallelism")]
26    pub share_buffers_sync_parallelism: u32,
27
28    /// Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use
29    /// tokio's default value (number of CPU core).
30    #[serde(default = "default::storage::share_buffer_compaction_worker_threads_number")]
31    pub share_buffer_compaction_worker_threads_number: u32,
32
33    /// Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity
34    /// will stall until there is enough space. The overridden value will only be effective if:
35    /// 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly.
36    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
37    #[serde(default)]
38    pub shared_buffer_capacity_mb: Option<usize>,
39
40    /// The shared buffer will start flushing data to object when the ratio of memory usage to the
41    /// shared buffer capacity exceed such ratio.
42    #[serde(default = "default::storage::shared_buffer_flush_ratio")]
43    pub shared_buffer_flush_ratio: f32,
44
45    /// The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger,
46    /// the total flush size across multiple epochs should be at least higher than this size.
47    #[serde(default = "default::storage::shared_buffer_min_batch_flush_size_mb")]
48    pub shared_buffer_min_batch_flush_size_mb: usize,
49
50    /// The threshold for the number of immutable memtables to merge to a new imm.
51    #[serde(default = "default::storage::imm_merge_threshold")]
52    #[deprecated]
53    pub imm_merge_threshold: usize,
54
55    /// Whether to enable write conflict detection
56    #[serde(default = "default::storage::write_conflict_detection_enabled")]
57    pub write_conflict_detection_enabled: bool,
58
59    #[serde(default)]
60    #[config_doc(nested)]
61    pub cache: CacheConfig,
62
63    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead.
64    #[serde(default)]
65    pub block_cache_capacity_mb: Option<usize>,
66
67    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead.
68    #[serde(default)]
69    pub meta_cache_capacity_mb: Option<usize>,
70
71    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead.
72    #[serde(default)]
73    pub high_priority_ratio_in_percent: Option<usize>,
74
75    /// max memory usage for large query
76    #[serde(default)]
77    pub prefetch_buffer_capacity_mb: Option<usize>,
78
79    #[serde(default = "default::storage::max_cached_recent_versions_number")]
80    pub max_cached_recent_versions_number: usize,
81
82    /// max prefetch block number
83    #[serde(
84        default = "default::storage::max_prefetch_block_number",
85        deserialize_with = "deserialize_max_prefetch_block_number"
86    )]
87    pub max_prefetch_block_number: usize,
88
89    #[serde(default = "default::storage::disable_remote_compactor")]
90    pub disable_remote_compactor: bool,
91
92    /// Number of tasks shared buffer can upload in parallel.
93    #[serde(default = "default::storage::share_buffer_upload_concurrency")]
94    pub share_buffer_upload_concurrency: usize,
95
96    #[serde(default)]
97    pub compactor_memory_limit_mb: Option<usize>,
98
99    /// Compactor calculates the maximum number of tasks that can be executed on the node based on
100    /// `worker_num` and `compactor_max_task_multiplier`.
101    /// `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier`
102    #[serde(default = "default::storage::compactor_max_task_multiplier")]
103    pub compactor_max_task_multiplier: f32,
104
105    /// The percentage of memory available when compactor is deployed separately.
106    /// `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion`
107    #[serde(default = "default::storage::compactor_memory_available_proportion")]
108    pub compactor_memory_available_proportion: f64,
109
110    /// Number of SST ids fetched from meta per RPC
111    #[serde(default = "default::storage::sstable_id_remote_fetch_number")]
112    pub sstable_id_remote_fetch_number: u32,
113
114    #[serde(default = "default::storage::min_sstable_size_mb")]
115    pub min_sstable_size_mb: u32,
116
117    #[serde(default)]
118    #[config_doc(nested)]
119    pub data_file_cache: FileCacheConfig,
120
121    #[serde(default)]
122    #[config_doc(nested)]
123    pub meta_file_cache: FileCacheConfig,
124
125    /// sst serde happens when a sst meta is written to meta disk cache.
126    /// Excluding the SST filter from serde can reduce the meta disk cache entry size
127    /// and reduce disk IO throughput at the cost of making the SST filter useless.
128    #[serde(default = "default::storage::sst_skip_bloom_filter_in_serde")]
129    pub sst_skip_bloom_filter_in_serde: bool,
130
131    #[serde(default)]
132    #[config_doc(nested)]
133    pub cache_refill: CacheRefillConfig,
134
135    /// Whether to enable streaming upload for sstable.
136    #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
137    pub min_sst_size_for_streaming_upload: u64,
138
139    #[serde(default = "default::storage::max_concurrent_compaction_task_number")]
140    pub max_concurrent_compaction_task_number: u64,
141
142    #[serde(default = "default::storage::max_preload_wait_time_mill")]
143    pub max_preload_wait_time_mill: u64,
144
145    #[serde(default = "default::storage::max_version_pinning_duration_sec")]
146    pub max_version_pinning_duration_sec: u64,
147
148    #[serde(default = "default::storage::compactor_max_sst_key_count")]
149    pub compactor_max_sst_key_count: u64,
150    // DEPRECATED: This config will be deprecated in the future version, use `storage.compactor_iter_max_io_retry_times` instead.
151    #[serde(default = "default::storage::compact_iter_recreate_timeout_ms")]
152    pub compact_iter_recreate_timeout_ms: u64,
153    #[serde(default = "default::storage::compactor_max_sst_size")]
154    pub compactor_max_sst_size: u64,
155    #[serde(default = "default::storage::enable_fast_compaction")]
156    pub enable_fast_compaction: bool,
157    #[serde(default = "default::storage::check_compaction_result")]
158    pub check_compaction_result: bool,
159    #[serde(default = "default::storage::max_preload_io_retry_times")]
160    pub max_preload_io_retry_times: usize,
161    #[serde(default = "default::storage::compactor_fast_max_compact_delete_ratio")]
162    pub compactor_fast_max_compact_delete_ratio: u32,
163    #[serde(default = "default::storage::compactor_fast_max_compact_task_size")]
164    pub compactor_fast_max_compact_task_size: u64,
165    #[serde(default = "default::storage::compactor_iter_max_io_retry_times")]
166    pub compactor_iter_max_io_retry_times: usize,
167
168    /// If set, block metadata keys will be shortened when their length exceeds this threshold.
169    /// This reduces `SSTable` metadata size by storing only the minimal distinguishing prefix.
170    /// - `None`: Disabled (default)
171    /// - `Some(n)`: Only shorten keys with length >= n bytes
172    #[serde(default = "default::storage::shorten_block_meta_key_threshold")]
173    pub shorten_block_meta_key_threshold: Option<usize>,
174
175    /// Deprecated: The window size of table info statistic history.
176    #[serde(default = "default::storage::table_info_statistic_history_times")]
177    #[deprecated]
178    pub table_info_statistic_history_times: usize,
179
180    #[serde(default, flatten)]
181    #[config_doc(omitted)]
182    pub unrecognized: Unrecognized<Self>,
183
184    /// The spill threshold for mem table.
185    #[serde(default = "default::storage::mem_table_spill_threshold")]
186    pub mem_table_spill_threshold: usize,
187
188    /// The concurrent uploading number of `SSTables` of builder
189    #[serde(default = "default::storage::compactor_concurrent_uploading_sst_count")]
190    pub compactor_concurrent_uploading_sst_count: Option<usize>,
191
192    #[serde(default = "default::storage::compactor_max_overlap_sst_count")]
193    pub compactor_max_overlap_sst_count: usize,
194
195    /// The maximum number of meta files that can be preloaded.
196    /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`.
197    /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient.
198    #[serde(default = "default::storage::compactor_max_preload_meta_file_count")]
199    pub compactor_max_preload_meta_file_count: usize,
200
201    #[serde(default = "default::storage::vector_file_block_size_kb")]
202    pub vector_file_block_size_kb: usize,
203
204    /// Object storage configuration
205    /// 1. General configuration
206    /// 2. Some special configuration of Backend
207    /// 3. Retry and timeout configuration
208    #[serde(default)]
209    pub object_store: ObjectStoreConfig,
210
211    #[serde(default = "default::storage::time_travel_version_cache_capacity")]
212    pub time_travel_version_cache_capacity: u64,
213
214    #[serde(default = "default::storage::table_change_log_cache_capacity")]
215    pub table_change_log_cache_capacity: u64,
216
217    // iceberg compaction
218    /// Estimated heap memory budget used to schedule tasks in the dedicated Iceberg compactor, in
219    /// megabytes. This controls admission only; it is not a hard `DataFusion` allocation limit.
220    /// When unset, the budget is derived from the compactor's available memory.
221    #[serde(default)]
222    pub iceberg_compaction_memory_limit_mb: Option<usize>,
223    #[serde(default = "default::storage::iceberg_compaction_enable_validate")]
224    pub iceberg_compaction_enable_validate: bool,
225    #[serde(default = "default::storage::iceberg_compaction_max_record_batch_rows")]
226    pub iceberg_compaction_max_record_batch_rows: usize,
227    #[serde(default = "default::storage::iceberg_compaction_min_size_per_partition_mb")]
228    pub iceberg_compaction_min_size_per_partition_mb: u32,
229    #[serde(default = "default::storage::iceberg_compaction_max_file_count_per_partition")]
230    pub iceberg_compaction_max_file_count_per_partition: u32,
231    /// DEPRECATED: This config will be deprecated in the future version.
232    /// Use sink config `compaction.write_parquet_max_row_group_rows` instead.
233    #[serde(default = "default::storage::iceberg_compaction_write_parquet_max_row_group_rows")]
234    #[deprecated(
235        note = "This config is deprecated. Use sink config `compaction.write_parquet_max_row_group_rows` instead."
236    )]
237    pub iceberg_compaction_write_parquet_max_row_group_rows: usize,
238
239    /// The ratio of iceberg compaction max parallelism to the number of CPU cores
240    #[serde(default = "default::storage::iceberg_compaction_task_parallelism_ratio")]
241    pub iceberg_compaction_task_parallelism_ratio: f32,
242    /// Whether to enable heuristic output parallelism in iceberg compaction.
243    #[serde(default = "default::storage::iceberg_compaction_enable_heuristic_output_parallelism")]
244    pub iceberg_compaction_enable_heuristic_output_parallelism: bool,
245    /// Maximum number of concurrent file close operations
246    #[serde(default = "default::storage::iceberg_compaction_max_concurrent_closes")]
247    pub iceberg_compaction_max_concurrent_closes: usize,
248    /// Whether to enable dynamic size estimation for iceberg compaction.
249    #[serde(default = "default::storage::iceberg_compaction_enable_dynamic_size_estimation")]
250    pub iceberg_compaction_enable_dynamic_size_estimation: bool,
251    /// The smoothing factor for size estimation in iceberg compaction.(default: 0.3)
252    #[serde(default = "default::storage::iceberg_compaction_size_estimation_smoothing_factor")]
253    pub iceberg_compaction_size_estimation_smoothing_factor: f64,
254    /// Multiplier for pending waiting parallelism budget for iceberg compaction task queue.
255    /// Effective pending budget = `ceil(max_task_parallelism * multiplier)`. Default 4.0.
256    /// Set < 1.0 to reduce buffering (may increase `PullTask` RPC frequency); set higher to batch more tasks.
257    #[serde(
258        default = "default::storage::iceberg_compaction_pending_parallelism_budget_multiplier"
259    )]
260    pub iceberg_compaction_pending_parallelism_budget_multiplier: f32,
261    /// Maximum number of Iceberg compaction tasks requested in one pull.
262    #[serde(default = "default::storage::iceberg_compaction_max_pull_task_count")]
263    pub iceberg_compaction_max_pull_task_count: u32,
264    /// Pull interval for iceberg compaction task requests in milliseconds.
265    #[serde(
266        default = "default::storage::iceberg_compaction_pull_interval_ms",
267        deserialize_with = "deserialize_iceberg_compaction_pull_interval_ms"
268    )]
269    pub iceberg_compaction_pull_interval_ms: u64,
270    /// Enable prefetching entire data files before compacting them.
271    ///
272    /// When enabled, each input file is downloaded with a single HTTP GET before compaction
273    /// begins, replacing the default pattern of N+1 range reads (1 footer + N column chunks)
274    /// with a single sequential read per file. This reduces object storage READ API calls
275    /// from D×(1+N) to D per compaction cycle.
276    ///
277    /// Trade-off: higher peak memory — one full file is held in memory per concurrent
278    /// compaction task. Enable only when object storage API cost is a priority and memory headroom is
279    /// sufficient. See also the memory-protection config knobs such as
280    /// `iceberg_compaction_task_parallelism_ratio`.
281    #[serde(default = "default::storage::iceberg_compaction_enable_prefetch")]
282    pub iceberg_compaction_enable_prefetch: bool,
283
284    #[serde(default = "default::storage::iceberg_compaction_target_binpack_group_size_mb")]
285    pub iceberg_compaction_target_binpack_group_size_mb: Option<u64>,
286    #[serde(default = "default::storage::iceberg_compaction_min_group_size_mb")]
287    pub iceberg_compaction_min_group_size_mb: Option<u64>,
288    #[serde(default = "default::storage::iceberg_compaction_min_group_file_count")]
289    pub iceberg_compaction_min_group_file_count: Option<usize>,
290}
291
292/// the section `[storage.cache]` in `risingwave.toml`.
293#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
294#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
295pub struct CacheConfig {
296    /// Configure the capacity of the block cache in MB explicitly.
297    /// The overridden value will only be effective if:
298    /// 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
299    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
300    #[serde(default)]
301    pub block_cache_capacity_mb: Option<usize>,
302
303    /// Configure the number of shards in the block cache explicitly.
304    /// If not set, the shard number will be determined automatically based on cache capacity.
305    #[serde(default)]
306    pub block_cache_shard_num: Option<usize>,
307
308    #[serde(default)]
309    #[config_doc(omitted)]
310    pub block_cache_eviction: CacheEvictionConfig,
311
312    /// Configure the capacity of the block cache in MB explicitly.
313    /// The overridden value will only be effective if:
314    /// 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
315    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
316    #[serde(default)]
317    pub meta_cache_capacity_mb: Option<usize>,
318
319    /// Configure the number of shards in the meta cache explicitly.
320    /// If not set, the shard number will be determined automatically based on cache capacity.
321    #[serde(default)]
322    pub meta_cache_shard_num: Option<usize>,
323
324    #[serde(default)]
325    #[config_doc(omitted)]
326    pub meta_cache_eviction: CacheEvictionConfig,
327
328    #[serde(default = "default::storage::vector_block_cache_capacity_mb")]
329    pub vector_block_cache_capacity_mb: usize,
330    #[serde(default = "default::storage::vector_block_cache_shard_num")]
331    pub vector_block_cache_shard_num: usize,
332    #[serde(default)]
333    #[config_doc(omitted)]
334    pub vector_block_cache_eviction_config: CacheEvictionConfig,
335    #[serde(default = "default::storage::vector_meta_cache_capacity_mb")]
336    pub vector_meta_cache_capacity_mb: usize,
337    #[serde(default = "default::storage::vector_meta_cache_shard_num")]
338    pub vector_meta_cache_shard_num: usize,
339    #[serde(default)]
340    #[config_doc(omitted)]
341    pub vector_meta_cache_eviction_config: CacheEvictionConfig,
342}
343
344/// the section `[storage.cache.eviction]` in `risingwave.toml`.
345#[derive(Clone, Debug, Serialize, Deserialize)]
346#[serde(tag = "algorithm")]
347pub enum CacheEvictionConfig {
348    Lru {
349        high_priority_ratio_in_percent: Option<usize>,
350    },
351    Lfu {
352        window_capacity_ratio_in_percent: Option<usize>,
353        protected_capacity_ratio_in_percent: Option<usize>,
354        cmsketch_eps: Option<f64>,
355        cmsketch_confidence: Option<f64>,
356    },
357    S3Fifo {
358        small_queue_capacity_ratio_in_percent: Option<usize>,
359        ghost_queue_capacity_ratio_in_percent: Option<usize>,
360        small_to_main_freq_threshold: Option<u8>,
361    },
362}
363
364impl Default for CacheEvictionConfig {
365    fn default() -> Self {
366        Self::Lru {
367            high_priority_ratio_in_percent: None,
368        }
369    }
370}
371
372#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
373#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
374pub struct CacheRefillConfig {
375    /// Inflight meta cache refill tasks limit.
376    ///
377    /// 0 for unlimited.
378    #[serde(default = "default::cache_refill::meta_refill_concurrency")]
379    pub meta_refill_concurrency: usize,
380
381    /// `SSTable` levels to refill.
382    #[serde(default = "default::cache_refill::data_refill_levels")]
383    pub data_refill_levels: Vec<u32>,
384
385    /// Cache refill maximum timeout to apply version delta.
386    #[serde(default = "default::cache_refill::timeout_ms")]
387    pub timeout_ms: u64,
388
389    /// Inflight data cache refill tasks.
390    #[serde(default = "default::cache_refill::concurrency")]
391    pub concurrency: usize,
392
393    /// Block count that a data cache refill request fetches.
394    #[serde(default = "default::cache_refill::unit")]
395    pub unit: usize,
396
397    /// Data cache refill unit admission ratio.
398    ///
399    /// Only unit whose blocks are admitted above the ratio will be refilled.
400    #[serde(default = "default::cache_refill::threshold")]
401    pub threshold: f64,
402
403    /// Recent filter layer shards.
404    #[serde(default = "default::cache_refill::recent_filter_shards")]
405    pub recent_filter_shards: usize,
406
407    /// Recent filter layer count.
408    #[serde(default = "default::cache_refill::recent_filter_layers")]
409    pub recent_filter_layers: usize,
410
411    /// Recent filter layer rotate interval.
412    #[serde(default = "default::cache_refill::recent_filter_rotate_interval_ms")]
413    pub recent_filter_rotate_interval_ms: usize,
414
415    /// Skip checking recent filter on data refill.
416    ///
417    /// This option is suitable for a single compute node or debugging.
418    #[serde(default = "default::cache_refill::skip_recent_filter")]
419    pub skip_recent_filter: bool,
420
421    /// Skip checking inheritance filter on data refill.
422    ///
423    /// The inheritance filter only runs after recent-filter admission, so this
424    /// option has no effect when `skip_recent_filter` is enabled.
425    ///
426    /// This option is suitable for a single compute node or debugging.
427    #[serde(default = "default::cache_refill::skip_inheritance_filter")]
428    pub skip_inheritance_filter: bool,
429
430    #[serde(default, flatten)]
431    #[config_doc(omitted)]
432    pub unrecognized: Unrecognized<Self>,
433}
434
435#[derive(Clone, Debug, Default, Serialize, Deserialize)]
436pub struct FileCacheTokioRuntimeConfig {
437    /// Dedicated runtime worker threads. `0` uses the Tokio default.
438    pub worker_threads: usize,
439
440    /// Maximum number of blocking threads. `0` uses the Tokio default.
441    pub max_blocking_threads: usize,
442}
443
444#[derive(Clone, Debug, Serialize, Deserialize)]
445pub enum FileCacheRuntimeConfig {
446    /// Use the runtime that creates the file cache.
447    Disabled,
448    /// Use one dedicated runtime for all file cache tasks.
449    Unified(FileCacheTokioRuntimeConfig),
450    /// Legacy configuration for separate read and write runtimes.
451    ///
452    /// Foyer 0.22 uses one spawner, so this configuration is no longer supported.
453    Separated {
454        read_runtime_options: FileCacheTokioRuntimeConfig,
455        write_runtime_options: FileCacheTokioRuntimeConfig,
456    },
457}
458
459/// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`.
460///
461/// It's put at [`StorageConfig::data_file_cache`] and  [`StorageConfig::meta_file_cache`].
462#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
463#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
464pub struct FileCacheConfig {
465    #[serde(default = "default::file_cache::dir")]
466    pub dir: String,
467
468    #[serde(default = "default::file_cache::capacity_mb")]
469    pub capacity_mb: usize,
470
471    #[serde(default = "default::file_cache::file_capacity_mb")]
472    pub file_capacity_mb: usize,
473
474    #[serde(default = "default::file_cache::flushers")]
475    pub flushers: usize,
476
477    #[serde(default = "default::file_cache::reclaimers")]
478    pub reclaimers: usize,
479
480    #[serde(default = "default::file_cache::recover_concurrency")]
481    pub recover_concurrency: usize,
482
483    /// Deprecated soon. Please use `throttle` to do I/O throttling instead.
484    #[serde(default = "default::file_cache::insert_rate_limit_mb")]
485    pub insert_rate_limit_mb: usize,
486
487    #[serde(default = "default::file_cache::indexer_shards")]
488    pub indexer_shards: usize,
489
490    #[serde(default = "default::file_cache::compression")]
491    pub compression: Compression,
492
493    #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
494    pub flush_buffer_threshold_mb: Option<usize>,
495
496    /// Maximum estimated size of cache entries waiting in the flusher submit queues. New cache
497    /// entries are ignored when the pending size exceeds this threshold.
498    #[serde(default = "default::file_cache::submit_queue_size_threshold_mb")]
499    pub submit_queue_size_threshold_mb: usize,
500
501    #[serde(default = "default::file_cache::throttle")]
502    pub throttle: Throttle,
503
504    #[serde(default = "default::file_cache::fifo_probation_ratio")]
505    pub fifo_probation_ratio: f64,
506
507    /// Set the blob index size for each blob.
508    ///
509    /// A larger blob index size can hold more blob entries, but it will also increase the io size of each blob part
510    /// write.
511    ///
512    /// NOTE:
513    ///
514    /// - The size will be aligned up to a multiplier of 4K.
515    /// - Modifying this configuration will invalidate all existing file cache data.
516    ///
517    /// Default: 16 `KiB`
518    #[serde(default = "default::file_cache::blob_index_size_kb")]
519    pub blob_index_size_kb: usize,
520
521    /// Recover mode.
522    ///
523    /// Options:
524    ///
525    /// - "None": Do not recover disk cache.
526    /// - "Quiet": Recover disk cache and skip errors.
527    /// - "Strict": Recover disk cache and panic on errors.
528    ///
529    /// More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`],
530    #[serde(default = "default::file_cache::recover_mode")]
531    pub recover_mode: RecoverMode,
532
533    #[serde(default = "default::file_cache::runtime_config")]
534    pub runtime_config: FileCacheRuntimeConfig,
535
536    #[serde(default, flatten)]
537    #[config_doc(omitted)]
538    pub unrecognized: Unrecognized<Self>,
539}
540
541/// The subsections `[storage.object_store]`.
542#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
543#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
544pub struct ObjectStoreConfig {
545    // alias is for backward compatibility
546    #[serde(
547        default = "default::object_store_config::set_atomic_write_dir",
548        alias = "object_store_set_atomic_write_dir"
549    )]
550    pub set_atomic_write_dir: bool,
551
552    /// Retry and timeout configuration
553    /// Description retry strategy driven by exponential back-off
554    /// Exposes the timeout and retries of each Object store interface. Therefore, the total timeout for each interface is determined based on the interface's timeout/retry configuration and the exponential back-off policy.
555    #[serde(default)]
556    pub retry: ObjectStoreRetryConfig,
557
558    /// Some special configuration of S3 Backend
559    #[serde(default)]
560    pub s3: S3ObjectStoreConfig,
561
562    /// Maximum number of concurrent object store requests (read, `streaming_read`, metadata, etc.).
563    /// 0 means unlimited. When set to a positive value, a semaphore will be used to limit
564    /// the number of in-flight requests to the object store, preventing HTTP connection pool
565    /// contention under high concurrency.
566    #[serde(default = "default::object_store_config::object_store_req_concurrency_limit")]
567    pub req_concurrency_limit: usize,
568
569    /// Maximum number of concurrent HTTP requests used by the `OpenDAL` GCS backend.
570    /// 0 means unlimited.
571    #[serde(default = "default::object_store_config::http_concurrent_limit")]
572    pub http_concurrent_limit: usize,
573
574    // TODO: the following field will be deprecated after opendal is stabilized
575    #[serde(default = "default::object_store_config::opendal_upload_concurrency")]
576    pub opendal_upload_concurrency: usize,
577
578    // TODO: the following field will be deprecated after opendal is stabilized
579    #[serde(default)]
580    pub opendal_writer_abort_on_err: bool,
581
582    #[serde(default = "default::object_store_config::upload_part_size")]
583    pub upload_part_size: usize,
584}
585
586fn deserialize_max_prefetch_block_number<'de, D>(deserializer: D) -> Result<usize, D::Error>
587where
588    D: serde::Deserializer<'de>,
589{
590    let value = usize::deserialize(deserializer)?;
591    if value == 0 {
592        return Err(D::Error::custom(
593            "storage.max_prefetch_block_number must be greater than 0",
594        ));
595    }
596    Ok(value)
597}
598
599fn deserialize_iceberg_compaction_pull_interval_ms<'de, D>(deserializer: D) -> Result<u64, D::Error>
600where
601    D: serde::Deserializer<'de>,
602{
603    let value = u64::deserialize(deserializer)?;
604    if value == 0 {
605        return Err(D::Error::custom(
606            "storage.iceberg_compaction_pull_interval_ms must be greater than 0",
607        ));
608    }
609    Ok(value)
610}
611
612impl ObjectStoreConfig {
613    pub fn set_atomic_write_dir(&mut self) {
614        self.set_atomic_write_dir = true;
615    }
616}
617
618/// The subsections `[storage.object_store.s3]`.
619#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
620#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
621pub struct S3ObjectStoreConfig {
622    // alias is for backward compatibility
623    #[serde(
624        default = "default::object_store_config::s3::keepalive_ms",
625        alias = "object_store_keepalive_ms"
626    )]
627    pub keepalive_ms: Option<u64>,
628    #[serde(
629        default = "default::object_store_config::s3::recv_buffer_size",
630        alias = "object_store_recv_buffer_size"
631    )]
632    pub recv_buffer_size: Option<usize>,
633    #[serde(
634        default = "default::object_store_config::s3::send_buffer_size",
635        alias = "object_store_send_buffer_size"
636    )]
637    pub send_buffer_size: Option<usize>,
638    #[serde(
639        default = "default::object_store_config::s3::nodelay",
640        alias = "object_store_nodelay"
641    )]
642    pub nodelay: Option<bool>,
643    /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead.
644    #[serde(default = "default::object_store_config::s3::developer::retry_unknown_service_error")]
645    pub retry_unknown_service_error: bool,
646    #[serde(default = "default::object_store_config::s3::identity_resolution_timeout_s")]
647    pub identity_resolution_timeout_s: u64,
648    #[serde(default)]
649    pub developer: S3ObjectStoreDeveloperConfig,
650}
651
652/// The subsections `[storage.object_store.s3.developer]`.
653#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
654#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
655pub struct S3ObjectStoreDeveloperConfig {
656    /// Whether to retry s3 sdk error from which no error metadata is provided.
657    #[serde(
658        default = "default::object_store_config::s3::developer::retry_unknown_service_error",
659        alias = "object_store_retry_unknown_service_error"
660    )]
661    pub retry_unknown_service_error: bool,
662    /// An array of error codes that should be retried.
663    /// e.g. `["SlowDown", "TooManyRequests"]`
664    #[serde(
665        default = "default::object_store_config::s3::developer::retryable_service_error_codes",
666        alias = "object_store_retryable_service_error_codes"
667    )]
668    pub retryable_service_error_codes: Vec<String>,
669
670    // TODO: deprecate this config when we are completely deprecate aws sdk.
671    #[serde(default = "default::object_store_config::s3::developer::use_opendal")]
672    pub use_opendal: bool,
673}
674
675#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
676#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
677pub struct ObjectStoreRetryConfig {
678    // A retry strategy driven by exponential back-off.
679    // The retry strategy is used for all object store operations.
680    /// Given a base duration for retry strategy in milliseconds.
681    #[serde(default = "default::object_store_config::object_store_req_backoff_interval_ms")]
682    pub req_backoff_interval_ms: u64,
683
684    /// The max delay interval for the retry strategy. No retry delay will be longer than this `Duration`.
685    #[serde(default = "default::object_store_config::object_store_req_backoff_max_delay_ms")]
686    pub req_backoff_max_delay_ms: u64,
687
688    /// A multiplicative factor that will be applied to the exponential back-off retry delay.
689    #[serde(default = "default::object_store_config::object_store_req_backoff_factor")]
690    pub req_backoff_factor: u64,
691
692    /// Maximum timeout for `upload` operation
693    #[serde(default = "default::object_store_config::object_store_upload_attempt_timeout_ms")]
694    pub upload_attempt_timeout_ms: u64,
695
696    /// Total counts of `upload` operation retries
697    #[serde(default = "default::object_store_config::object_store_upload_retry_attempts")]
698    pub upload_retry_attempts: usize,
699
700    /// Maximum timeout for `streaming_upload_init` and `streaming_upload`
701    #[serde(
702        default = "default::object_store_config::object_store_streaming_upload_attempt_timeout_ms"
703    )]
704    pub streaming_upload_attempt_timeout_ms: u64,
705
706    /// Total counts of `streaming_upload` operation retries
707    #[serde(
708        default = "default::object_store_config::object_store_streaming_upload_retry_attempts"
709    )]
710    pub streaming_upload_retry_attempts: usize,
711
712    /// Maximum timeout for `read` operation
713    #[serde(default = "default::object_store_config::object_store_read_attempt_timeout_ms")]
714    pub read_attempt_timeout_ms: u64,
715
716    /// Total counts of `read` operation retries
717    #[serde(default = "default::object_store_config::object_store_read_retry_attempts")]
718    pub read_retry_attempts: usize,
719
720    /// Maximum timeout for `streaming_read_init` and `streaming_read` operation
721    #[serde(
722        default = "default::object_store_config::object_store_streaming_read_attempt_timeout_ms"
723    )]
724    pub streaming_read_attempt_timeout_ms: u64,
725
726    /// Total counts of `streaming_read operation` retries
727    #[serde(default = "default::object_store_config::object_store_streaming_read_retry_attempts")]
728    pub streaming_read_retry_attempts: usize,
729
730    /// Maximum timeout for `metadata` operation
731    #[serde(default = "default::object_store_config::object_store_metadata_attempt_timeout_ms")]
732    pub metadata_attempt_timeout_ms: u64,
733
734    /// Total counts of `metadata` operation retries
735    #[serde(default = "default::object_store_config::object_store_metadata_retry_attempts")]
736    pub metadata_retry_attempts: usize,
737
738    /// Maximum timeout for `delete` operation
739    #[serde(default = "default::object_store_config::object_store_delete_attempt_timeout_ms")]
740    pub delete_attempt_timeout_ms: u64,
741
742    /// Total counts of `delete` operation retries
743    #[serde(default = "default::object_store_config::object_store_delete_retry_attempts")]
744    pub delete_retry_attempts: usize,
745
746    /// Maximum timeout for `delete_object` operation
747    #[serde(
748        default = "default::object_store_config::object_store_delete_objects_attempt_timeout_ms"
749    )]
750    pub delete_objects_attempt_timeout_ms: u64,
751
752    /// Total counts of `delete_object` operation retries
753    #[serde(default = "default::object_store_config::object_store_delete_objects_retry_attempts")]
754    pub delete_objects_retry_attempts: usize,
755
756    /// Maximum timeout for `list` operation
757    #[serde(default = "default::object_store_config::object_store_list_attempt_timeout_ms")]
758    pub list_attempt_timeout_ms: u64,
759
760    /// Total counts of `list` operation retries
761    #[serde(default = "default::object_store_config::object_store_list_retry_attempts")]
762    pub list_retry_attempts: usize,
763}
764
765#[derive(Debug, Clone)]
766pub enum EvictionConfig {
767    Lru(LruConfig),
768    Lfu(LfuConfig),
769    S3Fifo(S3FifoConfig),
770}
771
772impl EvictionConfig {
773    pub fn for_test() -> Self {
774        Self::Lru(LruConfig {
775            high_priority_pool_ratio: 0.0,
776        })
777    }
778}
779
780impl From<EvictionConfig> for foyer::EvictionConfig {
781    fn from(value: EvictionConfig) -> Self {
782        match value {
783            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
784            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
785            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
786        }
787    }
788}
789
790pub struct StorageMemoryConfig {
791    pub block_cache_capacity_mb: usize,
792    pub block_cache_shard_num: usize,
793    pub meta_cache_capacity_mb: usize,
794    pub meta_cache_shard_num: usize,
795    pub vector_block_cache_capacity_mb: usize,
796    pub vector_block_cache_shard_num: usize,
797    pub vector_meta_cache_capacity_mb: usize,
798    pub vector_meta_cache_shard_num: usize,
799    pub shared_buffer_capacity_mb: usize,
800    pub compactor_memory_limit_mb: usize,
801    pub prefetch_buffer_capacity_mb: usize,
802    pub block_cache_eviction_config: EvictionConfig,
803    pub meta_cache_eviction_config: EvictionConfig,
804    pub vector_block_cache_eviction_config: EvictionConfig,
805    pub vector_meta_cache_eviction_config: EvictionConfig,
806    pub block_file_cache_flush_buffer_threshold_mb: usize,
807    pub meta_file_cache_flush_buffer_threshold_mb: usize,
808}
809
810pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
811    let block_cache_capacity_mb = s.storage.cache.block_cache_capacity_mb.unwrap_or(
812        // adapt to old version
813        s.storage
814            .block_cache_capacity_mb
815            .unwrap_or(default::storage::block_cache_capacity_mb()),
816    );
817    let meta_cache_capacity_mb = s.storage.cache.meta_cache_capacity_mb.unwrap_or(
818        // adapt to old version
819        s.storage
820            .block_cache_capacity_mb
821            .unwrap_or(default::storage::meta_cache_capacity_mb()),
822    );
823    let shared_buffer_capacity_mb = s
824        .storage
825        .shared_buffer_capacity_mb
826        .unwrap_or(default::storage::shared_buffer_capacity_mb());
827    let meta_cache_shard_num = s.storage.cache.meta_cache_shard_num.unwrap_or_else(|| {
828        let mut shard_bits = MAX_META_CACHE_SHARD_BITS;
829        while (meta_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0 {
830            shard_bits -= 1;
831        }
832        shard_bits
833    });
834    let block_cache_shard_num = s.storage.cache.block_cache_shard_num.unwrap_or_else(|| {
835        let mut shard_bits = MAX_BLOCK_CACHE_SHARD_BITS;
836        while (block_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0
837        {
838            shard_bits -= 1;
839        }
840        shard_bits
841    });
842    let compactor_memory_limit_mb = s
843        .storage
844        .compactor_memory_limit_mb
845        .unwrap_or(default::storage::compactor_memory_limit_mb());
846
847    let get_eviction_config = |c: &CacheEvictionConfig| {
848        match c {
849            CacheEvictionConfig::Lru {
850                high_priority_ratio_in_percent,
851            } => EvictionConfig::Lru(LruConfig {
852                high_priority_pool_ratio: high_priority_ratio_in_percent.unwrap_or(
853                    // adapt to old version
854                    s.storage
855                        .high_priority_ratio_in_percent
856                        .unwrap_or(default::storage::high_priority_ratio_in_percent()),
857                ) as f64
858                    / 100.0,
859            }),
860            CacheEvictionConfig::Lfu {
861                window_capacity_ratio_in_percent,
862                protected_capacity_ratio_in_percent,
863                cmsketch_eps,
864                cmsketch_confidence,
865            } => EvictionConfig::Lfu(LfuConfig {
866                window_capacity_ratio: window_capacity_ratio_in_percent
867                    .unwrap_or(default::storage::window_capacity_ratio_in_percent())
868                    as f64
869                    / 100.0,
870                protected_capacity_ratio: protected_capacity_ratio_in_percent
871                    .unwrap_or(default::storage::protected_capacity_ratio_in_percent())
872                    as f64
873                    / 100.0,
874                cmsketch_eps: cmsketch_eps.unwrap_or(default::storage::cmsketch_eps()),
875                cmsketch_confidence: cmsketch_confidence
876                    .unwrap_or(default::storage::cmsketch_confidence()),
877            }),
878            CacheEvictionConfig::S3Fifo {
879                small_queue_capacity_ratio_in_percent,
880                ghost_queue_capacity_ratio_in_percent,
881                small_to_main_freq_threshold,
882            } => EvictionConfig::S3Fifo(S3FifoConfig {
883                small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
884                    .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
885                    as f64
886                    / 100.0,
887                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
888                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
889                    as f64
890                    / 100.0,
891                small_to_main_freq_threshold: small_to_main_freq_threshold
892                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
893            }),
894        }
895    };
896
897    let block_cache_eviction_config = get_eviction_config(&s.storage.cache.block_cache_eviction);
898    let meta_cache_eviction_config = get_eviction_config(&s.storage.cache.meta_cache_eviction);
899    let vector_block_cache_eviction_config =
900        get_eviction_config(&s.storage.cache.vector_block_cache_eviction_config);
901    let vector_meta_cache_eviction_config =
902        get_eviction_config(&s.storage.cache.vector_meta_cache_eviction_config);
903
904    let prefetch_buffer_capacity_mb =
905        s.storage
906            .shared_buffer_capacity_mb
907            .unwrap_or(match &block_cache_eviction_config {
908                EvictionConfig::Lru(lru) => {
909                    ((1.0 - lru.high_priority_pool_ratio) * block_cache_capacity_mb as f64) as usize
910                }
911                EvictionConfig::Lfu(lfu) => {
912                    ((1.0 - lfu.protected_capacity_ratio) * block_cache_capacity_mb as f64) as usize
913                }
914                EvictionConfig::S3Fifo(s3fifo) => {
915                    (s3fifo.small_queue_capacity_ratio * block_cache_capacity_mb as f64) as usize
916                }
917            });
918
919    let block_file_cache_flush_buffer_threshold_mb = s
920        .storage
921        .data_file_cache
922        .flush_buffer_threshold_mb
923        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
924    let meta_file_cache_flush_buffer_threshold_mb = s
925        .storage
926        .meta_file_cache
927        .flush_buffer_threshold_mb
928        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
929
930    StorageMemoryConfig {
931        block_cache_capacity_mb,
932        block_cache_shard_num,
933        meta_cache_capacity_mb,
934        meta_cache_shard_num,
935        vector_block_cache_capacity_mb: s.storage.cache.vector_block_cache_capacity_mb,
936        vector_block_cache_shard_num: s.storage.cache.vector_block_cache_shard_num,
937        vector_meta_cache_capacity_mb: s.storage.cache.vector_meta_cache_capacity_mb,
938        vector_meta_cache_shard_num: s.storage.cache.vector_meta_cache_shard_num,
939        shared_buffer_capacity_mb,
940        compactor_memory_limit_mb,
941        prefetch_buffer_capacity_mb,
942        block_cache_eviction_config,
943        meta_cache_eviction_config,
944        vector_block_cache_eviction_config,
945        vector_meta_cache_eviction_config,
946        block_file_cache_flush_buffer_threshold_mb,
947        meta_file_cache_flush_buffer_threshold_mb,
948    }
949}
950
951pub mod default {
952
953    pub mod storage {
954        pub fn share_buffers_sync_parallelism() -> u32 {
955            1
956        }
957
958        pub fn share_buffer_compaction_worker_threads_number() -> u32 {
959            4
960        }
961
962        pub fn shared_buffer_capacity_mb() -> usize {
963            1024
964        }
965
966        pub fn shared_buffer_flush_ratio() -> f32 {
967            0.8
968        }
969
970        pub fn shared_buffer_min_batch_flush_size_mb() -> usize {
971            800
972        }
973
974        pub fn imm_merge_threshold() -> usize {
975            0 // disable
976        }
977
978        pub fn write_conflict_detection_enabled() -> bool {
979            cfg!(debug_assertions)
980        }
981
982        pub fn max_cached_recent_versions_number() -> usize {
983            60
984        }
985
986        pub fn block_cache_capacity_mb() -> usize {
987            512
988        }
989
990        pub fn high_priority_ratio_in_percent() -> usize {
991            70
992        }
993
994        pub fn window_capacity_ratio_in_percent() -> usize {
995            10
996        }
997
998        pub fn protected_capacity_ratio_in_percent() -> usize {
999            80
1000        }
1001
1002        pub fn cmsketch_eps() -> f64 {
1003            0.002
1004        }
1005
1006        pub fn cmsketch_confidence() -> f64 {
1007            0.95
1008        }
1009
1010        pub fn small_queue_capacity_ratio_in_percent() -> usize {
1011            10
1012        }
1013
1014        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
1015            1000
1016        }
1017
1018        pub fn small_to_main_freq_threshold() -> u8 {
1019            1
1020        }
1021
1022        pub fn meta_cache_capacity_mb() -> usize {
1023            128
1024        }
1025
1026        pub fn disable_remote_compactor() -> bool {
1027            false
1028        }
1029
1030        pub fn share_buffer_upload_concurrency() -> usize {
1031            8
1032        }
1033
1034        pub fn compactor_memory_limit_mb() -> usize {
1035            512
1036        }
1037
1038        pub fn compactor_max_task_multiplier() -> f32 {
1039            3.0
1040        }
1041
1042        pub fn compactor_memory_available_proportion() -> f64 {
1043            0.8
1044        }
1045
1046        pub fn sstable_id_remote_fetch_number() -> u32 {
1047            10
1048        }
1049
1050        pub fn min_sstable_size_mb() -> u32 {
1051            32
1052        }
1053
1054        pub fn min_sst_size_for_streaming_upload() -> u64 {
1055            // 32MB
1056            32 * 1024 * 1024
1057        }
1058
1059        pub fn max_concurrent_compaction_task_number() -> u64 {
1060            16
1061        }
1062
1063        pub fn max_preload_wait_time_mill() -> u64 {
1064            0
1065        }
1066
1067        pub fn max_version_pinning_duration_sec() -> u64 {
1068            3 * 3600
1069        }
1070
1071        pub fn compactor_max_sst_key_count() -> u64 {
1072            2 * 1024 * 1024 // 200w
1073        }
1074
1075        pub fn compact_iter_recreate_timeout_ms() -> u64 {
1076            10 * 60 * 1000
1077        }
1078
1079        pub fn compactor_iter_max_io_retry_times() -> usize {
1080            8
1081        }
1082
1083        pub fn shorten_block_meta_key_threshold() -> Option<usize> {
1084            None
1085        }
1086
1087        pub fn compactor_max_sst_size() -> u64 {
1088            512 * 1024 * 1024 // 512m
1089        }
1090
1091        pub fn enable_fast_compaction() -> bool {
1092            true
1093        }
1094
1095        pub fn check_compaction_result() -> bool {
1096            false
1097        }
1098
1099        pub fn max_preload_io_retry_times() -> usize {
1100            3
1101        }
1102
1103        pub fn mem_table_spill_threshold() -> usize {
1104            4 << 20
1105        }
1106
1107        pub fn compactor_fast_max_compact_delete_ratio() -> u32 {
1108            40
1109        }
1110
1111        pub fn compactor_fast_max_compact_task_size() -> u64 {
1112            2 * 1024 * 1024 * 1024 // 2g
1113        }
1114
1115        pub fn max_prefetch_block_number() -> usize {
1116            16
1117        }
1118
1119        pub fn compactor_concurrent_uploading_sst_count() -> Option<usize> {
1120            None
1121        }
1122
1123        pub fn compactor_max_overlap_sst_count() -> usize {
1124            64
1125        }
1126
1127        pub fn compactor_max_preload_meta_file_count() -> usize {
1128            32
1129        }
1130
1131        pub fn vector_file_block_size_kb() -> usize {
1132            1024
1133        }
1134
1135        pub fn vector_block_cache_capacity_mb() -> usize {
1136            16
1137        }
1138
1139        pub fn vector_block_cache_shard_num() -> usize {
1140            16
1141        }
1142
1143        pub fn vector_meta_cache_capacity_mb() -> usize {
1144            16
1145        }
1146
1147        pub fn vector_meta_cache_shard_num() -> usize {
1148            16
1149        }
1150
1151        // deprecated
1152        pub fn table_info_statistic_history_times() -> usize {
1153            240
1154        }
1155
1156        pub fn block_file_cache_flush_buffer_threshold_mb() -> usize {
1157            256
1158        }
1159
1160        pub fn meta_file_cache_flush_buffer_threshold_mb() -> usize {
1161            64
1162        }
1163
1164        pub fn time_travel_version_cache_capacity() -> u64 {
1165            10
1166        }
1167
1168        pub fn table_change_log_cache_capacity() -> u64 {
1169            60
1170        }
1171
1172        pub fn sst_skip_bloom_filter_in_serde() -> bool {
1173            false
1174        }
1175
1176        pub fn iceberg_compaction_enable_validate() -> bool {
1177            false
1178        }
1179
1180        pub fn iceberg_compaction_max_record_batch_rows() -> usize {
1181            1024
1182        }
1183
1184        pub fn iceberg_compaction_write_parquet_max_row_group_rows() -> usize {
1185            1024 * 100 // 100k
1186        }
1187
1188        pub fn iceberg_compaction_min_size_per_partition_mb() -> u32 {
1189            1024
1190        }
1191
1192        pub fn iceberg_compaction_max_file_count_per_partition() -> u32 {
1193            32
1194        }
1195
1196        pub fn iceberg_compaction_task_parallelism_ratio() -> f32 {
1197            4.0
1198        }
1199
1200        pub fn iceberg_compaction_enable_heuristic_output_parallelism() -> bool {
1201            false
1202        }
1203
1204        pub fn iceberg_compaction_max_concurrent_closes() -> usize {
1205            8
1206        }
1207
1208        pub fn iceberg_compaction_enable_dynamic_size_estimation() -> bool {
1209            true
1210        }
1211
1212        pub fn iceberg_compaction_size_estimation_smoothing_factor() -> f64 {
1213            0.3
1214        }
1215
1216        pub fn iceberg_compaction_pending_parallelism_budget_multiplier() -> f32 {
1217            4.0
1218        }
1219
1220        pub fn iceberg_compaction_max_pull_task_count() -> u32 {
1221            1
1222        }
1223
1224        pub fn iceberg_compaction_pull_interval_ms() -> u64 {
1225            5000
1226        }
1227
1228        pub fn iceberg_compaction_enable_prefetch() -> bool {
1229            false
1230        }
1231
1232        pub fn iceberg_compaction_target_binpack_group_size_mb() -> Option<u64> {
1233            Some(100 * 1024) // 100GB
1234        }
1235
1236        pub fn iceberg_compaction_min_group_size_mb() -> Option<u64> {
1237            None
1238        }
1239
1240        pub fn iceberg_compaction_min_group_file_count() -> Option<usize> {
1241            None
1242        }
1243    }
1244
1245    pub mod file_cache {
1246        use std::num::NonZeroUsize;
1247
1248        use foyer::{Compression, RecoverMode, Throttle};
1249
1250        use super::super::{FileCacheRuntimeConfig, FileCacheTokioRuntimeConfig};
1251
1252        pub fn dir() -> String {
1253            "".to_owned()
1254        }
1255
1256        pub fn capacity_mb() -> usize {
1257            1024
1258        }
1259
1260        pub fn file_capacity_mb() -> usize {
1261            64
1262        }
1263
1264        pub fn flushers() -> usize {
1265            4
1266        }
1267
1268        pub fn reclaimers() -> usize {
1269            4
1270        }
1271
1272        pub fn recover_concurrency() -> usize {
1273            8
1274        }
1275
1276        pub fn insert_rate_limit_mb() -> usize {
1277            0
1278        }
1279
1280        pub fn indexer_shards() -> usize {
1281            64
1282        }
1283
1284        pub fn compression() -> Compression {
1285            Compression::None
1286        }
1287
1288        pub fn flush_buffer_threshold_mb() -> Option<usize> {
1289            None
1290        }
1291
1292        pub fn submit_queue_size_threshold_mb() -> usize {
1293            16
1294        }
1295
1296        pub fn fifo_probation_ratio() -> f64 {
1297            0.1
1298        }
1299
1300        pub fn blob_index_size_kb() -> usize {
1301            16
1302        }
1303
1304        pub fn recover_mode() -> RecoverMode {
1305            RecoverMode::Quiet
1306        }
1307
1308        pub fn runtime_config() -> FileCacheRuntimeConfig {
1309            FileCacheRuntimeConfig::Unified(FileCacheTokioRuntimeConfig::default())
1310        }
1311
1312        pub fn throttle() -> Throttle {
1313            Throttle::new()
1314                .with_iops_counter(foyer::IopsCounter::PerIoSize(
1315                    NonZeroUsize::new(128 * 1024).unwrap(),
1316                ))
1317                .with_read_iops(100000)
1318                .with_write_iops(100000)
1319                .with_write_throughput(1024 * 1024 * 1024)
1320                .with_read_throughput(1024 * 1024 * 1024)
1321        }
1322    }
1323
1324    pub mod cache_refill {
1325        pub fn meta_refill_concurrency() -> usize {
1326            0
1327        }
1328
1329        pub fn data_refill_levels() -> Vec<u32> {
1330            vec![]
1331        }
1332
1333        pub fn timeout_ms() -> u64 {
1334            6000
1335        }
1336
1337        pub fn concurrency() -> usize {
1338            10
1339        }
1340
1341        pub fn unit() -> usize {
1342            64
1343        }
1344
1345        pub fn threshold() -> f64 {
1346            0.5
1347        }
1348
1349        pub fn recent_filter_shards() -> usize {
1350            16
1351        }
1352
1353        pub fn recent_filter_layers() -> usize {
1354            6
1355        }
1356
1357        pub fn recent_filter_rotate_interval_ms() -> usize {
1358            10000
1359        }
1360
1361        pub fn skip_recent_filter() -> bool {
1362            false
1363        }
1364
1365        pub fn skip_inheritance_filter() -> bool {
1366            false
1367        }
1368    }
1369
1370    pub mod object_store_config {
1371        const DEFAULT_REQ_BACKOFF_INTERVAL_MS: u64 = 1000; // 1s
1372        const DEFAULT_REQ_BACKOFF_MAX_DELAY_MS: u64 = 10 * 1000; // 10s
1373        const DEFAULT_REQ_MAX_RETRY_ATTEMPTS: usize = 3;
1374
1375        pub fn set_atomic_write_dir() -> bool {
1376            false
1377        }
1378
1379        pub fn object_store_req_concurrency_limit() -> usize {
1380            0
1381        }
1382
1383        pub fn http_concurrent_limit() -> usize {
1384            0
1385        }
1386
1387        pub fn object_store_req_backoff_interval_ms() -> u64 {
1388            DEFAULT_REQ_BACKOFF_INTERVAL_MS
1389        }
1390
1391        pub fn object_store_req_backoff_max_delay_ms() -> u64 {
1392            DEFAULT_REQ_BACKOFF_MAX_DELAY_MS // 10s
1393        }
1394
1395        pub fn object_store_req_backoff_factor() -> u64 {
1396            2
1397        }
1398
1399        pub fn object_store_upload_attempt_timeout_ms() -> u64 {
1400            8 * 1000 // 8s
1401        }
1402
1403        pub fn object_store_upload_retry_attempts() -> usize {
1404            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1405        }
1406
1407        // init + upload_part + finish
1408        pub fn object_store_streaming_upload_attempt_timeout_ms() -> u64 {
1409            5 * 1000 // 5s
1410        }
1411
1412        pub fn object_store_streaming_upload_retry_attempts() -> usize {
1413            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1414        }
1415
1416        // tips: depend on block_size
1417        pub fn object_store_read_attempt_timeout_ms() -> u64 {
1418            8 * 1000 // 8s
1419        }
1420
1421        pub fn object_store_read_retry_attempts() -> usize {
1422            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1423        }
1424
1425        pub fn object_store_streaming_read_attempt_timeout_ms() -> u64 {
1426            3 * 1000 // 3s
1427        }
1428
1429        pub fn object_store_streaming_read_retry_attempts() -> usize {
1430            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1431        }
1432
1433        pub fn object_store_metadata_attempt_timeout_ms() -> u64 {
1434            60 * 1000 // 1min
1435        }
1436
1437        pub fn object_store_metadata_retry_attempts() -> usize {
1438            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1439        }
1440
1441        pub fn object_store_delete_attempt_timeout_ms() -> u64 {
1442            5 * 1000
1443        }
1444
1445        pub fn object_store_delete_retry_attempts() -> usize {
1446            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1447        }
1448
1449        // tips: depend on batch size
1450        pub fn object_store_delete_objects_attempt_timeout_ms() -> u64 {
1451            5 * 1000
1452        }
1453
1454        pub fn object_store_delete_objects_retry_attempts() -> usize {
1455            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1456        }
1457
1458        pub fn object_store_list_attempt_timeout_ms() -> u64 {
1459            10 * 60 * 1000
1460        }
1461
1462        pub fn object_store_list_retry_attempts() -> usize {
1463            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1464        }
1465
1466        pub fn opendal_upload_concurrency() -> usize {
1467            256
1468        }
1469
1470        pub fn upload_part_size() -> usize {
1471            // 16m
1472            16 * 1024 * 1024
1473        }
1474
1475        pub mod s3 {
1476            const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
1477
1478            const DEFAULT_KEEPALIVE_MS: u64 = 600 * 1000; // 10min
1479
1480            pub fn keepalive_ms() -> Option<u64> {
1481                Some(DEFAULT_KEEPALIVE_MS) // 10min
1482            }
1483
1484            pub fn recv_buffer_size() -> Option<usize> {
1485                Some(1 << 21) // 2m
1486            }
1487
1488            pub fn send_buffer_size() -> Option<usize> {
1489                None
1490            }
1491
1492            pub fn nodelay() -> Option<bool> {
1493                Some(true)
1494            }
1495
1496            pub fn identity_resolution_timeout_s() -> u64 {
1497                DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S
1498            }
1499
1500            pub mod developer {
1501                pub fn retry_unknown_service_error() -> bool {
1502                    false
1503                }
1504
1505                pub fn retryable_service_error_codes() -> Vec<String> {
1506                    vec!["SlowDown".into(), "TooManyRequests".into()]
1507                }
1508
1509                pub fn use_opendal() -> bool {
1510                    true
1511                }
1512            }
1513        }
1514    }
1515}
risingwave_common/config/storage.rs

risingwave_common/config/
storage.rs
