risingwave_common/config/
storage.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use foyer::{
16    Compression, LfuConfig, LruConfig, RecoverMode, RuntimeOptions, S3FifoConfig, Throttle,
17};
18
19use super::*;
20
21/// The section `[storage]` in `risingwave.toml`.
22#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
23pub struct StorageConfig {
24    /// parallelism while syncing share buffers into L0 SST. Should NOT be 0.
25    #[serde(default = "default::storage::share_buffers_sync_parallelism")]
26    pub share_buffers_sync_parallelism: u32,
27
28    /// Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use
29    /// tokio's default value (number of CPU core).
30    #[serde(default = "default::storage::share_buffer_compaction_worker_threads_number")]
31    pub share_buffer_compaction_worker_threads_number: u32,
32
33    /// Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity
34    /// will stall until there is enough space. The overridden value will only be effective if:
35    /// 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly.
36    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
37    #[serde(default)]
38    pub shared_buffer_capacity_mb: Option<usize>,
39
40    /// The shared buffer will start flushing data to object when the ratio of memory usage to the
41    /// shared buffer capacity exceed such ratio.
42    #[serde(default = "default::storage::shared_buffer_flush_ratio")]
43    pub shared_buffer_flush_ratio: f32,
44
45    /// The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger,
46    /// the total flush size across multiple epochs should be at least higher than this size.
47    #[serde(default = "default::storage::shared_buffer_min_batch_flush_size_mb")]
48    pub shared_buffer_min_batch_flush_size_mb: usize,
49
50    /// The threshold for the number of immutable memtables to merge to a new imm.
51    #[serde(default = "default::storage::imm_merge_threshold")]
52    #[deprecated]
53    pub imm_merge_threshold: usize,
54
55    /// Whether to enable write conflict detection
56    #[serde(default = "default::storage::write_conflict_detection_enabled")]
57    pub write_conflict_detection_enabled: bool,
58
59    #[serde(default)]
60    #[config_doc(nested)]
61    pub cache: CacheConfig,
62
63    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead.
64    #[serde(default)]
65    pub block_cache_capacity_mb: Option<usize>,
66
67    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead.
68    #[serde(default)]
69    pub meta_cache_capacity_mb: Option<usize>,
70
71    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead.
72    #[serde(default)]
73    pub high_priority_ratio_in_percent: Option<usize>,
74
75    /// max memory usage for large query
76    #[serde(default)]
77    pub prefetch_buffer_capacity_mb: Option<usize>,
78
79    #[serde(default = "default::storage::max_cached_recent_versions_number")]
80    pub max_cached_recent_versions_number: usize,
81
82    /// max prefetch block number
83    #[serde(default = "default::storage::max_prefetch_block_number")]
84    pub max_prefetch_block_number: usize,
85
86    #[serde(default = "default::storage::disable_remote_compactor")]
87    pub disable_remote_compactor: bool,
88
89    /// Number of tasks shared buffer can upload in parallel.
90    #[serde(default = "default::storage::share_buffer_upload_concurrency")]
91    pub share_buffer_upload_concurrency: usize,
92
93    #[serde(default)]
94    pub compactor_memory_limit_mb: Option<usize>,
95
96    /// Compactor calculates the maximum number of tasks that can be executed on the node based on
97    /// `worker_num` and `compactor_max_task_multiplier`.
98    /// `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier`
99    #[serde(default = "default::storage::compactor_max_task_multiplier")]
100    pub compactor_max_task_multiplier: f32,
101
102    /// The percentage of memory available when compactor is deployed separately.
103    /// `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion`
104    #[serde(default = "default::storage::compactor_memory_available_proportion")]
105    pub compactor_memory_available_proportion: f64,
106
107    /// Number of SST ids fetched from meta per RPC
108    #[serde(default = "default::storage::sstable_id_remote_fetch_number")]
109    pub sstable_id_remote_fetch_number: u32,
110
111    #[serde(default = "default::storage::min_sstable_size_mb")]
112    pub min_sstable_size_mb: u32,
113
114    #[serde(default)]
115    #[config_doc(nested)]
116    pub data_file_cache: FileCacheConfig,
117
118    #[serde(default)]
119    #[config_doc(nested)]
120    pub meta_file_cache: FileCacheConfig,
121
122    #[serde(default)]
123    #[config_doc(nested)]
124    pub cache_refill: CacheRefillConfig,
125
126    /// Whether to enable streaming upload for sstable.
127    #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
128    pub min_sst_size_for_streaming_upload: u64,
129
130    #[serde(default = "default::storage::max_concurrent_compaction_task_number")]
131    pub max_concurrent_compaction_task_number: u64,
132
133    #[serde(default = "default::storage::max_preload_wait_time_mill")]
134    pub max_preload_wait_time_mill: u64,
135
136    #[serde(default = "default::storage::max_version_pinning_duration_sec")]
137    pub max_version_pinning_duration_sec: u64,
138
139    #[serde(default = "default::storage::compactor_max_sst_key_count")]
140    pub compactor_max_sst_key_count: u64,
141    // DEPRECATED: This config will be deprecated in the future version, use `storage.compactor_iter_max_io_retry_times` instead.
142    #[serde(default = "default::storage::compact_iter_recreate_timeout_ms")]
143    pub compact_iter_recreate_timeout_ms: u64,
144    #[serde(default = "default::storage::compactor_max_sst_size")]
145    pub compactor_max_sst_size: u64,
146    #[serde(default = "default::storage::enable_fast_compaction")]
147    pub enable_fast_compaction: bool,
148    #[serde(default = "default::storage::check_compaction_result")]
149    pub check_compaction_result: bool,
150    #[serde(default = "default::storage::max_preload_io_retry_times")]
151    pub max_preload_io_retry_times: usize,
152    #[serde(default = "default::storage::compactor_fast_max_compact_delete_ratio")]
153    pub compactor_fast_max_compact_delete_ratio: u32,
154    #[serde(default = "default::storage::compactor_fast_max_compact_task_size")]
155    pub compactor_fast_max_compact_task_size: u64,
156    #[serde(default = "default::storage::compactor_iter_max_io_retry_times")]
157    pub compactor_iter_max_io_retry_times: usize,
158
159    /// Deprecated: The window size of table info statistic history.
160    #[serde(default = "default::storage::table_info_statistic_history_times")]
161    #[deprecated]
162    pub table_info_statistic_history_times: usize,
163
164    #[serde(default, flatten)]
165    #[config_doc(omitted)]
166    pub unrecognized: Unrecognized<Self>,
167
168    /// The spill threshold for mem table.
169    #[serde(default = "default::storage::mem_table_spill_threshold")]
170    pub mem_table_spill_threshold: usize,
171
172    /// The concurrent uploading number of `SSTables` of builder
173    #[serde(default = "default::storage::compactor_concurrent_uploading_sst_count")]
174    pub compactor_concurrent_uploading_sst_count: Option<usize>,
175
176    #[serde(default = "default::storage::compactor_max_overlap_sst_count")]
177    pub compactor_max_overlap_sst_count: usize,
178
179    /// The maximum number of meta files that can be preloaded.
180    /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`.
181    /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient.
182    #[serde(default = "default::storage::compactor_max_preload_meta_file_count")]
183    pub compactor_max_preload_meta_file_count: usize,
184
185    #[serde(default = "default::storage::vector_file_block_size_kb")]
186    pub vector_file_block_size_kb: usize,
187
188    /// Object storage configuration
189    /// 1. General configuration
190    /// 2. Some special configuration of Backend
191    /// 3. Retry and timeout configuration
192    #[serde(default)]
193    pub object_store: ObjectStoreConfig,
194
195    #[serde(default = "default::storage::time_travel_version_cache_capacity")]
196    pub time_travel_version_cache_capacity: u64,
197
198    // iceberg compaction
199    #[serde(default = "default::storage::iceberg_compaction_target_file_size_mb")]
200    pub iceberg_compaction_target_file_size_mb: u32,
201    #[serde(default = "default::storage::iceberg_compaction_enable_validate")]
202    pub iceberg_compaction_enable_validate: bool,
203    #[serde(default = "default::storage::iceberg_compaction_max_record_batch_rows")]
204    pub iceberg_compaction_max_record_batch_rows: usize,
205    #[serde(default = "default::storage::iceberg_compaction_min_size_per_partition_mb")]
206    pub iceberg_compaction_min_size_per_partition_mb: u32,
207    #[serde(default = "default::storage::iceberg_compaction_max_file_count_per_partition")]
208    pub iceberg_compaction_max_file_count_per_partition: u32,
209    #[serde(default = "default::storage::iceberg_compaction_write_parquet_max_row_group_rows")]
210    pub iceberg_compaction_write_parquet_max_row_group_rows: usize,
211
212    /// The ratio of iceberg compaction max parallelism to the number of CPU cores
213    #[serde(default = "default::storage::iceberg_compaction_task_parallelism_ratio")]
214    pub iceberg_compaction_task_parallelism_ratio: f32,
215    /// Whether to enable heuristic output parallelism in iceberg compaction.
216    #[serde(default = "default::storage::iceberg_compaction_enable_heuristic_output_parallelism")]
217    pub iceberg_compaction_enable_heuristic_output_parallelism: bool,
218    /// Maximum number of concurrent file close operations
219    #[serde(default = "default::storage::iceberg_compaction_max_concurrent_closes")]
220    pub iceberg_compaction_max_concurrent_closes: usize,
221    /// Whether to enable dynamic size estimation for iceberg compaction.
222    #[serde(default = "default::storage::iceberg_compaction_enable_dynamic_size_estimation")]
223    pub iceberg_compaction_enable_dynamic_size_estimation: bool,
224    /// The smoothing factor for size estimation in iceberg compaction.(default: 0.3)
225    #[serde(default = "default::storage::iceberg_compaction_size_estimation_smoothing_factor")]
226    pub iceberg_compaction_size_estimation_smoothing_factor: f64,
227    // For Small File Compaction
228    /// The threshold for small file compaction in MB.
229    #[serde(default = "default::storage::iceberg_compaction_small_file_threshold_mb")]
230    pub iceberg_compaction_small_file_threshold_mb: u32,
231    /// The maximum total size of tasks in small file compaction in MB.
232    #[serde(default = "default::storage::iceberg_compaction_max_task_total_size_mb")]
233    pub iceberg_compaction_max_task_total_size_mb: u32,
234}
235
236/// the section `[storage.cache]` in `risingwave.toml`.
237#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
238pub struct CacheConfig {
239    /// Configure the capacity of the block cache in MB explicitly.
240    /// The overridden value will only be effective if:
241    /// 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
242    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
243    #[serde(default)]
244    pub block_cache_capacity_mb: Option<usize>,
245
246    /// Configure the number of shards in the block cache explicitly.
247    /// If not set, the shard number will be determined automatically based on cache capacity.
248    #[serde(default)]
249    pub block_cache_shard_num: Option<usize>,
250
251    #[serde(default)]
252    #[config_doc(omitted)]
253    pub block_cache_eviction: CacheEvictionConfig,
254
255    /// Configure the capacity of the block cache in MB explicitly.
256    /// The overridden value will only be effective if:
257    /// 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
258    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
259    #[serde(default)]
260    pub meta_cache_capacity_mb: Option<usize>,
261
262    /// Configure the number of shards in the meta cache explicitly.
263    /// If not set, the shard number will be determined automatically based on cache capacity.
264    #[serde(default)]
265    pub meta_cache_shard_num: Option<usize>,
266
267    #[serde(default)]
268    #[config_doc(omitted)]
269    pub meta_cache_eviction: CacheEvictionConfig,
270
271    #[serde(default = "default::storage::vector_block_cache_capacity_mb")]
272    pub vector_block_cache_capacity_mb: usize,
273    #[serde(default = "default::storage::vector_block_cache_shard_num")]
274    pub vector_block_cache_shard_num: usize,
275    #[serde(default)]
276    #[config_doc(omitted)]
277    pub vector_block_cache_eviction_config: CacheEvictionConfig,
278    #[serde(default = "default::storage::vector_meta_cache_capacity_mb")]
279    pub vector_meta_cache_capacity_mb: usize,
280    #[serde(default = "default::storage::vector_meta_cache_shard_num")]
281    pub vector_meta_cache_shard_num: usize,
282    #[serde(default)]
283    #[config_doc(omitted)]
284    pub vector_meta_cache_eviction_config: CacheEvictionConfig,
285}
286
287/// the section `[storage.cache.eviction]` in `risingwave.toml`.
288#[derive(Clone, Debug, Serialize, Deserialize)]
289#[serde(tag = "algorithm")]
290pub enum CacheEvictionConfig {
291    Lru {
292        high_priority_ratio_in_percent: Option<usize>,
293    },
294    Lfu {
295        window_capacity_ratio_in_percent: Option<usize>,
296        protected_capacity_ratio_in_percent: Option<usize>,
297        cmsketch_eps: Option<f64>,
298        cmsketch_confidence: Option<f64>,
299    },
300    S3Fifo {
301        small_queue_capacity_ratio_in_percent: Option<usize>,
302        ghost_queue_capacity_ratio_in_percent: Option<usize>,
303        small_to_main_freq_threshold: Option<u8>,
304    },
305}
306
307impl Default for CacheEvictionConfig {
308    fn default() -> Self {
309        Self::Lru {
310            high_priority_ratio_in_percent: None,
311        }
312    }
313}
314
315#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
316pub struct CacheRefillConfig {
317    /// `SSTable` levels to refill.
318    #[serde(default = "default::cache_refill::data_refill_levels")]
319    pub data_refill_levels: Vec<u32>,
320
321    /// Cache refill maximum timeout to apply version delta.
322    #[serde(default = "default::cache_refill::timeout_ms")]
323    pub timeout_ms: u64,
324
325    /// Inflight data cache refill tasks.
326    #[serde(default = "default::cache_refill::concurrency")]
327    pub concurrency: usize,
328
329    /// Block count that a data cache refill request fetches.
330    #[serde(default = "default::cache_refill::unit")]
331    pub unit: usize,
332
333    /// Data cache refill unit admission ratio.
334    ///
335    /// Only unit whose blocks are admitted above the ratio will be refilled.
336    #[serde(default = "default::cache_refill::threshold")]
337    pub threshold: f64,
338
339    /// Recent filter layer shards.
340    #[serde(default = "default::cache_refill::recent_filter_shards")]
341    pub recent_filter_shards: usize,
342
343    /// Recent filter layer count.
344    #[serde(default = "default::cache_refill::recent_filter_layers")]
345    pub recent_filter_layers: usize,
346
347    /// Recent filter layer rotate interval.
348    #[serde(default = "default::cache_refill::recent_filter_rotate_interval_ms")]
349    pub recent_filter_rotate_interval_ms: usize,
350
351    /// Skip check recent filter on data refill.
352    ///
353    /// This option is suitable for a single compute node or debugging.
354    #[serde(default = "default::cache_refill::skip_recent_filter")]
355    pub skip_recent_filter: bool,
356
357    #[serde(default, flatten)]
358    #[config_doc(omitted)]
359    pub unrecognized: Unrecognized<Self>,
360}
361
362/// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`.
363///
364/// It's put at [`StorageConfig::data_file_cache`] and  [`StorageConfig::meta_file_cache`].
365#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
366pub struct FileCacheConfig {
367    #[serde(default = "default::file_cache::dir")]
368    pub dir: String,
369
370    #[serde(default = "default::file_cache::capacity_mb")]
371    pub capacity_mb: usize,
372
373    #[serde(default = "default::file_cache::file_capacity_mb")]
374    pub file_capacity_mb: usize,
375
376    #[serde(default = "default::file_cache::flushers")]
377    pub flushers: usize,
378
379    #[serde(default = "default::file_cache::reclaimers")]
380    pub reclaimers: usize,
381
382    #[serde(default = "default::file_cache::recover_concurrency")]
383    pub recover_concurrency: usize,
384
385    /// Deprecated soon. Please use `throttle` to do I/O throttling instead.
386    #[serde(default = "default::file_cache::insert_rate_limit_mb")]
387    pub insert_rate_limit_mb: usize,
388
389    #[serde(default = "default::file_cache::indexer_shards")]
390    pub indexer_shards: usize,
391
392    #[serde(default = "default::file_cache::compression")]
393    pub compression: Compression,
394
395    #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
396    pub flush_buffer_threshold_mb: Option<usize>,
397
398    #[serde(default = "default::file_cache::throttle")]
399    pub throttle: Throttle,
400
401    #[serde(default = "default::file_cache::fifo_probation_ratio")]
402    pub fifo_probation_ratio: f64,
403
404    /// Set the blob index size for each blob.
405    ///
406    /// A larger blob index size can hold more blob entries, but it will also increase the io size of each blob part
407    /// write.
408    ///
409    /// NOTE:
410    ///
411    /// - The size will be aligned up to a multiplier of 4K.
412    /// - Modifying this configuration will invalidate all existing file cache data.
413    ///
414    /// Default: 16 `KiB`
415    #[serde(default = "default::file_cache::blob_index_size_kb")]
416    pub blob_index_size_kb: usize,
417
418    /// Recover mode.
419    ///
420    /// Options:
421    ///
422    /// - "None": Do not recover disk cache.
423    /// - "Quiet": Recover disk cache and skip errors.
424    /// - "Strict": Recover disk cache and panic on errors.
425    ///
426    /// More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`],
427    #[serde(default = "default::file_cache::recover_mode")]
428    pub recover_mode: RecoverMode,
429
430    #[serde(default = "default::file_cache::runtime_config")]
431    pub runtime_config: RuntimeOptions,
432
433    #[serde(default, flatten)]
434    #[config_doc(omitted)]
435    pub unrecognized: Unrecognized<Self>,
436}
437
438/// The subsections `[storage.object_store]`.
439#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
440pub struct ObjectStoreConfig {
441    // alias is for backward compatibility
442    #[serde(
443        default = "default::object_store_config::set_atomic_write_dir",
444        alias = "object_store_set_atomic_write_dir"
445    )]
446    pub set_atomic_write_dir: bool,
447
448    /// Retry and timeout configuration
449    /// Description retry strategy driven by exponential back-off
450    /// Exposes the timeout and retries of each Object store interface. Therefore, the total timeout for each interface is determined based on the interface's timeout/retry configuration and the exponential back-off policy.
451    #[serde(default)]
452    pub retry: ObjectStoreRetryConfig,
453
454    /// Some special configuration of S3 Backend
455    #[serde(default)]
456    pub s3: S3ObjectStoreConfig,
457
458    // TODO: the following field will be deprecated after opendal is stabilized
459    #[serde(default = "default::object_store_config::opendal_upload_concurrency")]
460    pub opendal_upload_concurrency: usize,
461
462    // TODO: the following field will be deprecated after opendal is stabilized
463    #[serde(default)]
464    pub opendal_writer_abort_on_err: bool,
465
466    #[serde(default = "default::object_store_config::upload_part_size")]
467    pub upload_part_size: usize,
468}
469
470impl ObjectStoreConfig {
471    pub fn set_atomic_write_dir(&mut self) {
472        self.set_atomic_write_dir = true;
473    }
474}
475
476/// The subsections `[storage.object_store.s3]`.
477#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
478pub struct S3ObjectStoreConfig {
479    // alias is for backward compatibility
480    #[serde(
481        default = "default::object_store_config::s3::keepalive_ms",
482        alias = "object_store_keepalive_ms"
483    )]
484    pub keepalive_ms: Option<u64>,
485    #[serde(
486        default = "default::object_store_config::s3::recv_buffer_size",
487        alias = "object_store_recv_buffer_size"
488    )]
489    pub recv_buffer_size: Option<usize>,
490    #[serde(
491        default = "default::object_store_config::s3::send_buffer_size",
492        alias = "object_store_send_buffer_size"
493    )]
494    pub send_buffer_size: Option<usize>,
495    #[serde(
496        default = "default::object_store_config::s3::nodelay",
497        alias = "object_store_nodelay"
498    )]
499    pub nodelay: Option<bool>,
500    /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead.
501    #[serde(default = "default::object_store_config::s3::developer::retry_unknown_service_error")]
502    pub retry_unknown_service_error: bool,
503    #[serde(default = "default::object_store_config::s3::identity_resolution_timeout_s")]
504    pub identity_resolution_timeout_s: u64,
505    #[serde(default)]
506    pub developer: S3ObjectStoreDeveloperConfig,
507}
508
509/// The subsections `[storage.object_store.s3.developer]`.
510#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
511pub struct S3ObjectStoreDeveloperConfig {
512    /// Whether to retry s3 sdk error from which no error metadata is provided.
513    #[serde(
514        default = "default::object_store_config::s3::developer::retry_unknown_service_error",
515        alias = "object_store_retry_unknown_service_error"
516    )]
517    pub retry_unknown_service_error: bool,
518    /// An array of error codes that should be retried.
519    /// e.g. `["SlowDown", "TooManyRequests"]`
520    #[serde(
521        default = "default::object_store_config::s3::developer::retryable_service_error_codes",
522        alias = "object_store_retryable_service_error_codes"
523    )]
524    pub retryable_service_error_codes: Vec<String>,
525
526    // TODO: deprecate this config when we are completely deprecate aws sdk.
527    #[serde(default = "default::object_store_config::s3::developer::use_opendal")]
528    pub use_opendal: bool,
529}
530
531#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
532pub struct ObjectStoreRetryConfig {
533    // A retry strategy driven by exponential back-off.
534    // The retry strategy is used for all object store operations.
535    /// Given a base duration for retry strategy in milliseconds.
536    #[serde(default = "default::object_store_config::object_store_req_backoff_interval_ms")]
537    pub req_backoff_interval_ms: u64,
538
539    /// The max delay interval for the retry strategy. No retry delay will be longer than this `Duration`.
540    #[serde(default = "default::object_store_config::object_store_req_backoff_max_delay_ms")]
541    pub req_backoff_max_delay_ms: u64,
542
543    /// A multiplicative factor that will be applied to the exponential back-off retry delay.
544    #[serde(default = "default::object_store_config::object_store_req_backoff_factor")]
545    pub req_backoff_factor: u64,
546
547    /// Maximum timeout for `upload` operation
548    #[serde(default = "default::object_store_config::object_store_upload_attempt_timeout_ms")]
549    pub upload_attempt_timeout_ms: u64,
550
551    /// Total counts of `upload` operation retries
552    #[serde(default = "default::object_store_config::object_store_upload_retry_attempts")]
553    pub upload_retry_attempts: usize,
554
555    /// Maximum timeout for `streaming_upload_init` and `streaming_upload`
556    #[serde(
557        default = "default::object_store_config::object_store_streaming_upload_attempt_timeout_ms"
558    )]
559    pub streaming_upload_attempt_timeout_ms: u64,
560
561    /// Total counts of `streaming_upload` operation retries
562    #[serde(
563        default = "default::object_store_config::object_store_streaming_upload_retry_attempts"
564    )]
565    pub streaming_upload_retry_attempts: usize,
566
567    /// Maximum timeout for `read` operation
568    #[serde(default = "default::object_store_config::object_store_read_attempt_timeout_ms")]
569    pub read_attempt_timeout_ms: u64,
570
571    /// Total counts of `read` operation retries
572    #[serde(default = "default::object_store_config::object_store_read_retry_attempts")]
573    pub read_retry_attempts: usize,
574
575    /// Maximum timeout for `streaming_read_init` and `streaming_read` operation
576    #[serde(
577        default = "default::object_store_config::object_store_streaming_read_attempt_timeout_ms"
578    )]
579    pub streaming_read_attempt_timeout_ms: u64,
580
581    /// Total counts of `streaming_read operation` retries
582    #[serde(default = "default::object_store_config::object_store_streaming_read_retry_attempts")]
583    pub streaming_read_retry_attempts: usize,
584
585    /// Maximum timeout for `metadata` operation
586    #[serde(default = "default::object_store_config::object_store_metadata_attempt_timeout_ms")]
587    pub metadata_attempt_timeout_ms: u64,
588
589    /// Total counts of `metadata` operation retries
590    #[serde(default = "default::object_store_config::object_store_metadata_retry_attempts")]
591    pub metadata_retry_attempts: usize,
592
593    /// Maximum timeout for `delete` operation
594    #[serde(default = "default::object_store_config::object_store_delete_attempt_timeout_ms")]
595    pub delete_attempt_timeout_ms: u64,
596
597    /// Total counts of `delete` operation retries
598    #[serde(default = "default::object_store_config::object_store_delete_retry_attempts")]
599    pub delete_retry_attempts: usize,
600
601    /// Maximum timeout for `delete_object` operation
602    #[serde(
603        default = "default::object_store_config::object_store_delete_objects_attempt_timeout_ms"
604    )]
605    pub delete_objects_attempt_timeout_ms: u64,
606
607    /// Total counts of `delete_object` operation retries
608    #[serde(default = "default::object_store_config::object_store_delete_objects_retry_attempts")]
609    pub delete_objects_retry_attempts: usize,
610
611    /// Maximum timeout for `list` operation
612    #[serde(default = "default::object_store_config::object_store_list_attempt_timeout_ms")]
613    pub list_attempt_timeout_ms: u64,
614
615    /// Total counts of `list` operation retries
616    #[serde(default = "default::object_store_config::object_store_list_retry_attempts")]
617    pub list_retry_attempts: usize,
618}
619
620#[derive(Debug, Clone)]
621pub enum EvictionConfig {
622    Lru(LruConfig),
623    Lfu(LfuConfig),
624    S3Fifo(S3FifoConfig),
625}
626
627impl EvictionConfig {
628    pub fn for_test() -> Self {
629        Self::Lru(LruConfig {
630            high_priority_pool_ratio: 0.0,
631        })
632    }
633}
634
635impl From<EvictionConfig> for foyer::EvictionConfig {
636    fn from(value: EvictionConfig) -> Self {
637        match value {
638            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
639            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
640            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
641        }
642    }
643}
644
645pub struct StorageMemoryConfig {
646    pub block_cache_capacity_mb: usize,
647    pub block_cache_shard_num: usize,
648    pub meta_cache_capacity_mb: usize,
649    pub meta_cache_shard_num: usize,
650    pub vector_block_cache_capacity_mb: usize,
651    pub vector_block_cache_shard_num: usize,
652    pub vector_meta_cache_capacity_mb: usize,
653    pub vector_meta_cache_shard_num: usize,
654    pub shared_buffer_capacity_mb: usize,
655    pub compactor_memory_limit_mb: usize,
656    pub prefetch_buffer_capacity_mb: usize,
657    pub block_cache_eviction_config: EvictionConfig,
658    pub meta_cache_eviction_config: EvictionConfig,
659    pub vector_block_cache_eviction_config: EvictionConfig,
660    pub vector_meta_cache_eviction_config: EvictionConfig,
661    pub block_file_cache_flush_buffer_threshold_mb: usize,
662    pub meta_file_cache_flush_buffer_threshold_mb: usize,
663}
664
665pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
666    let block_cache_capacity_mb = s.storage.cache.block_cache_capacity_mb.unwrap_or(
667        // adapt to old version
668        s.storage
669            .block_cache_capacity_mb
670            .unwrap_or(default::storage::block_cache_capacity_mb()),
671    );
672    let meta_cache_capacity_mb = s.storage.cache.meta_cache_capacity_mb.unwrap_or(
673        // adapt to old version
674        s.storage
675            .block_cache_capacity_mb
676            .unwrap_or(default::storage::meta_cache_capacity_mb()),
677    );
678    let shared_buffer_capacity_mb = s
679        .storage
680        .shared_buffer_capacity_mb
681        .unwrap_or(default::storage::shared_buffer_capacity_mb());
682    let meta_cache_shard_num = s.storage.cache.meta_cache_shard_num.unwrap_or_else(|| {
683        let mut shard_bits = MAX_META_CACHE_SHARD_BITS;
684        while (meta_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0 {
685            shard_bits -= 1;
686        }
687        shard_bits
688    });
689    let block_cache_shard_num = s.storage.cache.block_cache_shard_num.unwrap_or_else(|| {
690        let mut shard_bits = MAX_BLOCK_CACHE_SHARD_BITS;
691        while (block_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0
692        {
693            shard_bits -= 1;
694        }
695        shard_bits
696    });
697    let compactor_memory_limit_mb = s
698        .storage
699        .compactor_memory_limit_mb
700        .unwrap_or(default::storage::compactor_memory_limit_mb());
701
702    let get_eviction_config = |c: &CacheEvictionConfig| {
703        match c {
704            CacheEvictionConfig::Lru {
705                high_priority_ratio_in_percent,
706            } => EvictionConfig::Lru(LruConfig {
707                high_priority_pool_ratio: high_priority_ratio_in_percent.unwrap_or(
708                    // adapt to old version
709                    s.storage
710                        .high_priority_ratio_in_percent
711                        .unwrap_or(default::storage::high_priority_ratio_in_percent()),
712                ) as f64
713                    / 100.0,
714            }),
715            CacheEvictionConfig::Lfu {
716                window_capacity_ratio_in_percent,
717                protected_capacity_ratio_in_percent,
718                cmsketch_eps,
719                cmsketch_confidence,
720            } => EvictionConfig::Lfu(LfuConfig {
721                window_capacity_ratio: window_capacity_ratio_in_percent
722                    .unwrap_or(default::storage::window_capacity_ratio_in_percent())
723                    as f64
724                    / 100.0,
725                protected_capacity_ratio: protected_capacity_ratio_in_percent
726                    .unwrap_or(default::storage::protected_capacity_ratio_in_percent())
727                    as f64
728                    / 100.0,
729                cmsketch_eps: cmsketch_eps.unwrap_or(default::storage::cmsketch_eps()),
730                cmsketch_confidence: cmsketch_confidence
731                    .unwrap_or(default::storage::cmsketch_confidence()),
732            }),
733            CacheEvictionConfig::S3Fifo {
734                small_queue_capacity_ratio_in_percent,
735                ghost_queue_capacity_ratio_in_percent,
736                small_to_main_freq_threshold,
737            } => EvictionConfig::S3Fifo(S3FifoConfig {
738                small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
739                    .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
740                    as f64
741                    / 100.0,
742                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
743                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
744                    as f64
745                    / 100.0,
746                small_to_main_freq_threshold: small_to_main_freq_threshold
747                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
748            }),
749        }
750    };
751
752    let block_cache_eviction_config = get_eviction_config(&s.storage.cache.block_cache_eviction);
753    let meta_cache_eviction_config = get_eviction_config(&s.storage.cache.meta_cache_eviction);
754    let vector_block_cache_eviction_config =
755        get_eviction_config(&s.storage.cache.vector_block_cache_eviction_config);
756    let vector_meta_cache_eviction_config =
757        get_eviction_config(&s.storage.cache.vector_meta_cache_eviction_config);
758
759    let prefetch_buffer_capacity_mb =
760        s.storage
761            .shared_buffer_capacity_mb
762            .unwrap_or(match &block_cache_eviction_config {
763                EvictionConfig::Lru(lru) => {
764                    ((1.0 - lru.high_priority_pool_ratio) * block_cache_capacity_mb as f64) as usize
765                }
766                EvictionConfig::Lfu(lfu) => {
767                    ((1.0 - lfu.protected_capacity_ratio) * block_cache_capacity_mb as f64) as usize
768                }
769                EvictionConfig::S3Fifo(s3fifo) => {
770                    (s3fifo.small_queue_capacity_ratio * block_cache_capacity_mb as f64) as usize
771                }
772            });
773
774    let block_file_cache_flush_buffer_threshold_mb = s
775        .storage
776        .data_file_cache
777        .flush_buffer_threshold_mb
778        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
779    let meta_file_cache_flush_buffer_threshold_mb = s
780        .storage
781        .meta_file_cache
782        .flush_buffer_threshold_mb
783        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
784
785    StorageMemoryConfig {
786        block_cache_capacity_mb,
787        block_cache_shard_num,
788        meta_cache_capacity_mb,
789        meta_cache_shard_num,
790        vector_block_cache_capacity_mb: s.storage.cache.vector_block_cache_capacity_mb,
791        vector_block_cache_shard_num: s.storage.cache.vector_block_cache_shard_num,
792        vector_meta_cache_capacity_mb: s.storage.cache.vector_meta_cache_capacity_mb,
793        vector_meta_cache_shard_num: s.storage.cache.vector_meta_cache_shard_num,
794        shared_buffer_capacity_mb,
795        compactor_memory_limit_mb,
796        prefetch_buffer_capacity_mb,
797        block_cache_eviction_config,
798        meta_cache_eviction_config,
799        vector_block_cache_eviction_config,
800        vector_meta_cache_eviction_config,
801        block_file_cache_flush_buffer_threshold_mb,
802        meta_file_cache_flush_buffer_threshold_mb,
803    }
804}
805
806pub mod default {
807
808    pub mod storage {
809        pub fn share_buffers_sync_parallelism() -> u32 {
810            1
811        }
812
813        pub fn share_buffer_compaction_worker_threads_number() -> u32 {
814            4
815        }
816
817        pub fn shared_buffer_capacity_mb() -> usize {
818            1024
819        }
820
821        pub fn shared_buffer_flush_ratio() -> f32 {
822            0.8
823        }
824
825        pub fn shared_buffer_min_batch_flush_size_mb() -> usize {
826            800
827        }
828
829        pub fn imm_merge_threshold() -> usize {
830            0 // disable
831        }
832
833        pub fn write_conflict_detection_enabled() -> bool {
834            cfg!(debug_assertions)
835        }
836
837        pub fn max_cached_recent_versions_number() -> usize {
838            60
839        }
840
841        pub fn block_cache_capacity_mb() -> usize {
842            512
843        }
844
845        pub fn high_priority_ratio_in_percent() -> usize {
846            70
847        }
848
849        pub fn window_capacity_ratio_in_percent() -> usize {
850            10
851        }
852
853        pub fn protected_capacity_ratio_in_percent() -> usize {
854            80
855        }
856
857        pub fn cmsketch_eps() -> f64 {
858            0.002
859        }
860
861        pub fn cmsketch_confidence() -> f64 {
862            0.95
863        }
864
865        pub fn small_queue_capacity_ratio_in_percent() -> usize {
866            10
867        }
868
869        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
870            1000
871        }
872
873        pub fn small_to_main_freq_threshold() -> u8 {
874            1
875        }
876
877        pub fn meta_cache_capacity_mb() -> usize {
878            128
879        }
880
881        pub fn disable_remote_compactor() -> bool {
882            false
883        }
884
885        pub fn share_buffer_upload_concurrency() -> usize {
886            8
887        }
888
889        pub fn compactor_memory_limit_mb() -> usize {
890            512
891        }
892
893        pub fn compactor_max_task_multiplier() -> f32 {
894            match std::env::var("RW_COMPACTOR_MODE")
895                .unwrap_or_default()
896                .as_str()
897            {
898                mode if mode.contains("iceberg") => 12.0000,
899                _ => 3.0000,
900            }
901        }
902
903        pub fn compactor_memory_available_proportion() -> f64 {
904            0.8
905        }
906
907        pub fn sstable_id_remote_fetch_number() -> u32 {
908            10
909        }
910
911        pub fn min_sstable_size_mb() -> u32 {
912            32
913        }
914
915        pub fn min_sst_size_for_streaming_upload() -> u64 {
916            // 32MB
917            32 * 1024 * 1024
918        }
919
920        pub fn max_concurrent_compaction_task_number() -> u64 {
921            16
922        }
923
924        pub fn max_preload_wait_time_mill() -> u64 {
925            0
926        }
927
928        pub fn max_version_pinning_duration_sec() -> u64 {
929            3 * 3600
930        }
931
932        pub fn compactor_max_sst_key_count() -> u64 {
933            2 * 1024 * 1024 // 200w
934        }
935
936        pub fn compact_iter_recreate_timeout_ms() -> u64 {
937            10 * 60 * 1000
938        }
939
940        pub fn compactor_iter_max_io_retry_times() -> usize {
941            8
942        }
943
944        pub fn compactor_max_sst_size() -> u64 {
945            512 * 1024 * 1024 // 512m
946        }
947
948        pub fn enable_fast_compaction() -> bool {
949            true
950        }
951
952        pub fn check_compaction_result() -> bool {
953            false
954        }
955
956        pub fn max_preload_io_retry_times() -> usize {
957            3
958        }
959
960        pub fn mem_table_spill_threshold() -> usize {
961            4 << 20
962        }
963
964        pub fn compactor_fast_max_compact_delete_ratio() -> u32 {
965            40
966        }
967
968        pub fn compactor_fast_max_compact_task_size() -> u64 {
969            2 * 1024 * 1024 * 1024 // 2g
970        }
971
972        pub fn max_prefetch_block_number() -> usize {
973            16
974        }
975
976        pub fn compactor_concurrent_uploading_sst_count() -> Option<usize> {
977            None
978        }
979
980        pub fn compactor_max_overlap_sst_count() -> usize {
981            64
982        }
983
984        pub fn compactor_max_preload_meta_file_count() -> usize {
985            32
986        }
987
988        pub fn vector_file_block_size_kb() -> usize {
989            1024
990        }
991
992        pub fn vector_block_cache_capacity_mb() -> usize {
993            16
994        }
995
996        pub fn vector_block_cache_shard_num() -> usize {
997            16
998        }
999
1000        pub fn vector_meta_cache_capacity_mb() -> usize {
1001            16
1002        }
1003
1004        pub fn vector_meta_cache_shard_num() -> usize {
1005            16
1006        }
1007
1008        // deprecated
1009        pub fn table_info_statistic_history_times() -> usize {
1010            240
1011        }
1012
1013        pub fn block_file_cache_flush_buffer_threshold_mb() -> usize {
1014            256
1015        }
1016
1017        pub fn meta_file_cache_flush_buffer_threshold_mb() -> usize {
1018            64
1019        }
1020
1021        pub fn time_travel_version_cache_capacity() -> u64 {
1022            10
1023        }
1024
1025        pub fn iceberg_compaction_target_file_size_mb() -> u32 {
1026            1024
1027        }
1028
1029        pub fn iceberg_compaction_enable_validate() -> bool {
1030            false
1031        }
1032
1033        pub fn iceberg_compaction_max_record_batch_rows() -> usize {
1034            1024
1035        }
1036
1037        pub fn iceberg_compaction_write_parquet_max_row_group_rows() -> usize {
1038            1024 * 100 // 100k
1039        }
1040
1041        pub fn iceberg_compaction_min_size_per_partition_mb() -> u32 {
1042            1024
1043        }
1044
1045        pub fn iceberg_compaction_max_file_count_per_partition() -> u32 {
1046            32
1047        }
1048
1049        pub fn iceberg_compaction_task_parallelism_ratio() -> f32 {
1050            4.0
1051        }
1052
1053        pub fn iceberg_compaction_enable_heuristic_output_parallelism() -> bool {
1054            false
1055        }
1056
1057        pub fn iceberg_compaction_max_concurrent_closes() -> usize {
1058            8
1059        }
1060
1061        pub fn iceberg_compaction_enable_dynamic_size_estimation() -> bool {
1062            true
1063        }
1064
1065        pub fn iceberg_compaction_size_estimation_smoothing_factor() -> f64 {
1066            0.3
1067        }
1068
1069        pub fn iceberg_compaction_small_file_threshold_mb() -> u32 {
1070            32
1071        }
1072
1073        pub fn iceberg_compaction_max_task_total_size_mb() -> u32 {
1074            50 * 1024 // 50GB
1075        }
1076    }
1077
1078    pub mod file_cache {
1079        use std::num::NonZeroUsize;
1080
1081        use foyer::{Compression, RecoverMode, RuntimeOptions, Throttle, TokioRuntimeOptions};
1082
1083        pub fn dir() -> String {
1084            "".to_owned()
1085        }
1086
1087        pub fn capacity_mb() -> usize {
1088            1024
1089        }
1090
1091        pub fn file_capacity_mb() -> usize {
1092            64
1093        }
1094
1095        pub fn flushers() -> usize {
1096            4
1097        }
1098
1099        pub fn reclaimers() -> usize {
1100            4
1101        }
1102
1103        pub fn recover_concurrency() -> usize {
1104            8
1105        }
1106
1107        pub fn insert_rate_limit_mb() -> usize {
1108            0
1109        }
1110
1111        pub fn indexer_shards() -> usize {
1112            64
1113        }
1114
1115        pub fn compression() -> Compression {
1116            Compression::None
1117        }
1118
1119        pub fn flush_buffer_threshold_mb() -> Option<usize> {
1120            None
1121        }
1122
1123        pub fn fifo_probation_ratio() -> f64 {
1124            0.1
1125        }
1126
1127        pub fn blob_index_size_kb() -> usize {
1128            16
1129        }
1130
1131        pub fn recover_mode() -> RecoverMode {
1132            RecoverMode::Quiet
1133        }
1134
1135        pub fn runtime_config() -> RuntimeOptions {
1136            RuntimeOptions::Unified(TokioRuntimeOptions::default())
1137        }
1138
1139        pub fn throttle() -> Throttle {
1140            Throttle::new()
1141                .with_iops_counter(foyer::IopsCounter::PerIoSize(
1142                    NonZeroUsize::new(128 * 1024).unwrap(),
1143                ))
1144                .with_read_iops(100000)
1145                .with_write_iops(100000)
1146                .with_write_throughput(1024 * 1024 * 1024)
1147                .with_read_throughput(1024 * 1024 * 1024)
1148        }
1149    }
1150
1151    pub mod cache_refill {
1152        pub fn data_refill_levels() -> Vec<u32> {
1153            vec![]
1154        }
1155
1156        pub fn timeout_ms() -> u64 {
1157            6000
1158        }
1159
1160        pub fn concurrency() -> usize {
1161            10
1162        }
1163
1164        pub fn unit() -> usize {
1165            64
1166        }
1167
1168        pub fn threshold() -> f64 {
1169            0.5
1170        }
1171
1172        pub fn recent_filter_shards() -> usize {
1173            16
1174        }
1175
1176        pub fn recent_filter_layers() -> usize {
1177            6
1178        }
1179
1180        pub fn recent_filter_rotate_interval_ms() -> usize {
1181            10000
1182        }
1183
1184        pub fn skip_recent_filter() -> bool {
1185            false
1186        }
1187    }
1188
1189    pub mod object_store_config {
1190        const DEFAULT_REQ_BACKOFF_INTERVAL_MS: u64 = 1000; // 1s
1191        const DEFAULT_REQ_BACKOFF_MAX_DELAY_MS: u64 = 10 * 1000; // 10s
1192        const DEFAULT_REQ_MAX_RETRY_ATTEMPTS: usize = 3;
1193
1194        pub fn set_atomic_write_dir() -> bool {
1195            false
1196        }
1197
1198        pub fn object_store_req_backoff_interval_ms() -> u64 {
1199            DEFAULT_REQ_BACKOFF_INTERVAL_MS
1200        }
1201
1202        pub fn object_store_req_backoff_max_delay_ms() -> u64 {
1203            DEFAULT_REQ_BACKOFF_MAX_DELAY_MS // 10s
1204        }
1205
1206        pub fn object_store_req_backoff_factor() -> u64 {
1207            2
1208        }
1209
1210        pub fn object_store_upload_attempt_timeout_ms() -> u64 {
1211            8 * 1000 // 8s
1212        }
1213
1214        pub fn object_store_upload_retry_attempts() -> usize {
1215            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1216        }
1217
1218        // init + upload_part + finish
1219        pub fn object_store_streaming_upload_attempt_timeout_ms() -> u64 {
1220            5 * 1000 // 5s
1221        }
1222
1223        pub fn object_store_streaming_upload_retry_attempts() -> usize {
1224            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1225        }
1226
1227        // tips: depend on block_size
1228        pub fn object_store_read_attempt_timeout_ms() -> u64 {
1229            8 * 1000 // 8s
1230        }
1231
1232        pub fn object_store_read_retry_attempts() -> usize {
1233            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1234        }
1235
1236        pub fn object_store_streaming_read_attempt_timeout_ms() -> u64 {
1237            3 * 1000 // 3s
1238        }
1239
1240        pub fn object_store_streaming_read_retry_attempts() -> usize {
1241            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1242        }
1243
1244        pub fn object_store_metadata_attempt_timeout_ms() -> u64 {
1245            60 * 1000 // 1min
1246        }
1247
1248        pub fn object_store_metadata_retry_attempts() -> usize {
1249            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1250        }
1251
1252        pub fn object_store_delete_attempt_timeout_ms() -> u64 {
1253            5 * 1000
1254        }
1255
1256        pub fn object_store_delete_retry_attempts() -> usize {
1257            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1258        }
1259
1260        // tips: depend on batch size
1261        pub fn object_store_delete_objects_attempt_timeout_ms() -> u64 {
1262            5 * 1000
1263        }
1264
1265        pub fn object_store_delete_objects_retry_attempts() -> usize {
1266            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1267        }
1268
1269        pub fn object_store_list_attempt_timeout_ms() -> u64 {
1270            10 * 60 * 1000
1271        }
1272
1273        pub fn object_store_list_retry_attempts() -> usize {
1274            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
1275        }
1276
1277        pub fn opendal_upload_concurrency() -> usize {
1278            256
1279        }
1280
1281        pub fn upload_part_size() -> usize {
1282            // 16m
1283            16 * 1024 * 1024
1284        }
1285
1286        pub mod s3 {
1287            const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
1288
1289            const DEFAULT_KEEPALIVE_MS: u64 = 600 * 1000; // 10min
1290
1291            pub fn keepalive_ms() -> Option<u64> {
1292                Some(DEFAULT_KEEPALIVE_MS) // 10min
1293            }
1294
1295            pub fn recv_buffer_size() -> Option<usize> {
1296                Some(1 << 21) // 2m
1297            }
1298
1299            pub fn send_buffer_size() -> Option<usize> {
1300                None
1301            }
1302
1303            pub fn nodelay() -> Option<bool> {
1304                Some(true)
1305            }
1306
1307            pub fn identity_resolution_timeout_s() -> u64 {
1308                DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S
1309            }
1310
1311            pub mod developer {
1312                pub fn retry_unknown_service_error() -> bool {
1313                    false
1314                }
1315
1316                pub fn retryable_service_error_codes() -> Vec<String> {
1317                    vec!["SlowDown".into(), "TooManyRequests".into()]
1318                }
1319
1320                pub fn use_opendal() -> bool {
1321                    true
1322                }
1323            }
1324        }
1325    }
1326}
risingwave_common/config/storage.rs

risingwave_common/config/
storage.rs
