risingwave_common/
config.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! This module defines the structure of the configuration file `risingwave.toml`.
16//!
17//! [`RwConfig`] corresponds to the whole config file and each other config struct corresponds to a
18//! section in `risingwave.toml`.
19
20use std::collections::BTreeMap;
21use std::fs;
22use std::num::NonZeroUsize;
23
24use anyhow::Context;
25use clap::ValueEnum;
26use educe::Educe;
27use foyer::{
28    Compression, LfuConfig, LruConfig, RecoverMode, RuntimeOptions, S3FifoConfig, Throttle,
29};
30use risingwave_common_proc_macro::ConfigDoc;
31pub use risingwave_common_proc_macro::OverrideConfig;
32use risingwave_pb::meta::SystemParams;
33use serde::{Deserialize, Serialize, Serializer};
34use serde_default::DefaultFromSerde;
35use serde_json::Value;
36
37use crate::for_all_params;
38
39/// Use the maximum value for HTTP/2 connection window size to avoid deadlock among multiplexed
40/// streams on the same connection.
41pub const MAX_CONNECTION_WINDOW_SIZE: u32 = (1 << 31) - 1;
42/// Use a large value for HTTP/2 stream window size to improve the performance of remote exchange,
43/// as we don't rely on this for back-pressure.
44pub const STREAM_WINDOW_SIZE: u32 = 32 * 1024 * 1024; // 32 MB
45
46/// Unrecognized fields in a config section. Generic over the config section type to provide better
47/// error messages.
48///
49/// The current implementation will log warnings if there are unrecognized fields.
50#[derive(Educe)]
51#[educe(Clone, Default)]
52pub struct Unrecognized<T: 'static> {
53    inner: BTreeMap<String, Value>,
54    _marker: std::marker::PhantomData<&'static T>,
55}
56
57impl<T> std::fmt::Debug for Unrecognized<T> {
58    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        self.inner.fmt(f)
60    }
61}
62
63impl<T> Unrecognized<T> {
64    /// Returns all unrecognized fields as a map.
65    pub fn into_inner(self) -> BTreeMap<String, Value> {
66        self.inner
67    }
68}
69
70impl<'de, T> Deserialize<'de> for Unrecognized<T> {
71    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
72    where
73        D: serde::Deserializer<'de>,
74    {
75        let inner = BTreeMap::deserialize(deserializer)?;
76        if !inner.is_empty() {
77            tracing::warn!(
78                "unrecognized fields in `{}`: {:?}",
79                std::any::type_name::<T>(),
80                inner.keys()
81            );
82        }
83        Ok(Unrecognized {
84            inner,
85            _marker: std::marker::PhantomData,
86        })
87    }
88}
89
90impl<T> Serialize for Unrecognized<T> {
91    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
92    where
93        S: serde::Serializer,
94    {
95        self.inner.serialize(serializer)
96    }
97}
98
99pub fn load_config(path: &str, cli_override: impl OverrideConfig) -> RwConfig
100where
101{
102    let mut config = if path.is_empty() {
103        tracing::warn!("risingwave.toml not found, using default config.");
104        RwConfig::default()
105    } else {
106        let config_str = fs::read_to_string(path)
107            .with_context(|| format!("failed to open config file at `{path}`"))
108            .unwrap();
109        toml::from_str(config_str.as_str())
110            .context("failed to parse config file")
111            .unwrap()
112    };
113    cli_override.r#override(&mut config);
114    config
115}
116
117pub trait OverrideConfig {
118    fn r#override(&self, config: &mut RwConfig);
119}
120
121impl<T: OverrideConfig> OverrideConfig for &T {
122    fn r#override(&self, config: &mut RwConfig) {
123        T::r#override(self, config)
124    }
125}
126
127/// For non-user-facing components where the CLI arguments do not override the config file.
128#[derive(Clone, Copy)]
129pub struct NoOverride;
130
131impl OverrideConfig for NoOverride {
132    fn r#override(&self, _config: &mut RwConfig) {}
133}
134
135/// [`RwConfig`] corresponds to the whole config file `risingwave.toml`. Each field corresponds to a
136/// section.
137#[derive(Educe, Clone, Serialize, Deserialize, Default, ConfigDoc)]
138#[educe(Debug)]
139pub struct RwConfig {
140    #[serde(default)]
141    #[config_doc(nested)]
142    pub server: ServerConfig,
143
144    #[serde(default)]
145    #[config_doc(nested)]
146    pub meta: MetaConfig,
147
148    #[serde(default)]
149    #[config_doc(nested)]
150    pub batch: BatchConfig,
151
152    #[serde(default)]
153    #[config_doc(nested)]
154    pub frontend: FrontendConfig,
155
156    #[serde(default)]
157    #[config_doc(nested)]
158    pub streaming: StreamingConfig,
159
160    #[serde(default)]
161    #[config_doc(nested)]
162    pub storage: StorageConfig,
163
164    #[serde(default)]
165    #[educe(Debug(ignore))]
166    #[config_doc(nested)]
167    pub system: SystemConfig,
168
169    #[serde(default)]
170    #[config_doc(nested)]
171    pub udf: UdfConfig,
172
173    #[serde(flatten)]
174    #[config_doc(omitted)]
175    pub unrecognized: Unrecognized<Self>,
176}
177
178serde_with::with_prefix!(meta_prefix "meta_");
179serde_with::with_prefix!(streaming_prefix "stream_");
180serde_with::with_prefix!(batch_prefix "batch_");
181
182#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
183pub enum MetaBackend {
184    #[default]
185    Mem,
186    Sql, // any database url
187    Sqlite,
188    Postgres,
189    Mysql,
190}
191
192/// The section `[meta]` in `risingwave.toml`.
193#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
194pub struct MetaConfig {
195    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
196    /// are dangling.
197    #[serde(default = "default::meta::min_sst_retention_time_sec")]
198    pub min_sst_retention_time_sec: u64,
199
200    /// Interval of automatic hummock full GC.
201    #[serde(default = "default::meta::full_gc_interval_sec")]
202    pub full_gc_interval_sec: u64,
203
204    /// Max number of object per full GC job can fetch.
205    #[serde(default = "default::meta::full_gc_object_limit")]
206    pub full_gc_object_limit: u64,
207
208    /// Duration in seconds to retain garbage collection history data.
209    #[serde(default = "default::meta::gc_history_retention_time_sec")]
210    pub gc_history_retention_time_sec: u64,
211
212    /// Max number of inflight time travel query.
213    #[serde(default = "default::meta::max_inflight_time_travel_query")]
214    pub max_inflight_time_travel_query: u64,
215
216    /// Schedule compaction for all compaction groups with this interval.
217    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
218    pub periodic_compaction_interval_sec: u64,
219
220    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
221    /// from object store.
222    #[serde(default = "default::meta::vacuum_interval_sec")]
223    pub vacuum_interval_sec: u64,
224
225    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
226    /// meta node.
227    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
228    pub vacuum_spin_interval_ms: u64,
229
230    /// Interval of hummock version checkpoint.
231    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
232    pub hummock_version_checkpoint_interval_sec: u64,
233
234    /// If enabled, `SSTable` object file and version delta will be retained.
235    ///
236    /// `SSTable` object file need to be deleted via full GC.
237    ///
238    /// version delta need to be manually deleted.
239    #[serde(default = "default::meta::enable_hummock_data_archive")]
240    pub enable_hummock_data_archive: bool,
241
242    /// The interval at which a Hummock version snapshot is taken for time travel.
243    ///
244    /// Larger value indicates less storage overhead but worse query performance.
245    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
246    pub hummock_time_travel_snapshot_interval: u64,
247
248    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
249    /// attempt is rejected.
250    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
251    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
252
253    /// Maximum allowed heartbeat interval in seconds.
254    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
255    pub max_heartbeat_interval_secs: u32,
256
257    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
258    #[serde(default)]
259    pub disable_recovery: bool,
260
261    /// Whether to disable adaptive-scaling feature.
262    #[serde(default)]
263    pub disable_automatic_parallelism_control: bool,
264
265    /// The number of streaming jobs per scaling operation.
266    #[serde(default = "default::meta::parallelism_control_batch_size")]
267    pub parallelism_control_batch_size: usize,
268
269    /// The period of parallelism control trigger.
270    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
271    pub parallelism_control_trigger_period_sec: u64,
272
273    /// The first delay of parallelism control.
274    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
275    pub parallelism_control_trigger_first_delay_sec: u64,
276
277    #[serde(default = "default::meta::meta_leader_lease_secs")]
278    pub meta_leader_lease_secs: u64,
279
280    /// After specified seconds of idle (no mview or flush), the process will be exited.
281    /// It is mainly useful for playgrounds.
282    #[serde(default)]
283    pub dangerous_max_idle_secs: Option<u64>,
284
285    /// The default global parallelism for all streaming jobs, if user doesn't specify the
286    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
287    /// otherwise it's a number.
288    #[serde(default = "default::meta::default_parallelism")]
289    pub default_parallelism: DefaultParallelism,
290
291    /// Whether to enable deterministic compaction scheduling, which
292    /// will disable all auto scheduling of compaction tasks.
293    /// Should only be used in e2e tests.
294    #[serde(default)]
295    pub enable_compaction_deterministic: bool,
296
297    /// Enable sanity check when SSTs are committed.
298    #[serde(default)]
299    pub enable_committed_sst_sanity_check: bool,
300
301    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
302    pub node_num_monitor_interval_sec: u64,
303
304    #[serde(default = "default::meta::backend")]
305    pub backend: MetaBackend,
306
307    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
308    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
309    pub periodic_space_reclaim_compaction_interval_sec: u64,
310
311    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
312    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
313    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
314
315    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
316    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
317
318    #[serde(default = "default::meta::move_table_size_limit")]
319    #[deprecated]
320    pub move_table_size_limit: u64,
321
322    #[serde(default = "default::meta::split_group_size_limit")]
323    #[deprecated]
324    pub split_group_size_limit: u64,
325
326    #[serde(default = "default::meta::cut_table_size_limit")]
327    #[deprecated]
328    pub cut_table_size_limit: u64,
329
330    /// Whether to protect dropping a table with incoming sink.
331    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
332    pub protect_drop_table_with_incoming_sink: bool,
333
334    #[serde(default, flatten)]
335    #[config_doc(omitted)]
336    pub unrecognized: Unrecognized<Self>,
337
338    /// Whether config object storage bucket lifecycle to purge stale data.
339    #[serde(default)]
340    pub do_not_config_object_storage_lifecycle: bool,
341
342    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
343    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
344    #[serde(default = "default::meta::partition_vnode_count")]
345    pub partition_vnode_count: u32,
346
347    /// The threshold of write throughput to trigger a group split.
348    #[serde(
349        default = "default::meta::table_high_write_throughput_threshold",
350        alias = "table_write_throughput_threshold"
351    )]
352    pub table_high_write_throughput_threshold: u64,
353
354    #[serde(
355        default = "default::meta::table_low_write_throughput_threshold",
356        alias = "min_table_split_write_throughput"
357    )]
358    /// The threshold of write throughput to trigger a group merge.
359    pub table_low_write_throughput_threshold: u64,
360
361    // If the compaction task does not report heartbeat beyond the
362    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
363    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
364    pub compaction_task_max_heartbeat_interval_secs: u64,
365
366    // If the compaction task does not change in progress beyond the
367    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
368    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
369    pub compaction_task_max_progress_interval_secs: u64,
370
371    #[serde(default)]
372    #[config_doc(nested)]
373    pub compaction_config: CompactionConfig,
374
375    /// Count of partitions of tables in default group and materialized view group.
376    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
377    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
378    /// Set it zero to disable this feature.
379    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
380    pub hybrid_partition_vnode_count: u32,
381
382    #[serde(default = "default::meta::event_log_enabled")]
383    pub event_log_enabled: bool,
384    /// Keeps the latest N events per channel.
385    #[serde(default = "default::meta::event_log_channel_max_size")]
386    pub event_log_channel_max_size: u32,
387
388    #[serde(default, with = "meta_prefix")]
389    #[config_doc(omitted)]
390    pub developer: MetaDeveloperConfig,
391    /// Whether compactor should rewrite row to remove dropped column.
392    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
393    pub enable_dropped_column_reclaim: bool,
394
395    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
396    #[serde(default = "default::meta::split_group_size_ratio")]
397    pub split_group_size_ratio: f64,
398
399    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
400    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
401    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
402    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
403    pub table_stat_high_write_throughput_ratio_for_split: f64,
404
405    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
406    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
407    pub table_stat_low_write_throughput_ratio_for_merge: f64,
408
409    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
410    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
411    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
412    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
413    /// The window seconds of table throughput statistic history for split compaction group.
414    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
415    pub table_stat_throuput_window_seconds_for_split: usize,
416
417    /// The window seconds of table throughput statistic history for merge compaction group.
418    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
419    pub table_stat_throuput_window_seconds_for_merge: usize,
420
421    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
422    /// Set it max value of 64-bit number to disable this feature.
423    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
424    pub compact_task_table_size_partition_threshold_low: u64,
425
426    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
427    /// Set it max value of 64-bit number to disable this feature.
428    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
429    pub compact_task_table_size_partition_threshold_high: u64,
430
431    /// The interval of the periodic scheduling compaction group split job.
432    #[serde(
433        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
434        alias = "periodic_split_compact_group_interval_sec"
435    )]
436    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
437
438    /// The interval of the periodic scheduling compaction group merge job.
439    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
440    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
441
442    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
443    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
444    pub compaction_group_merge_dimension_threshold: f64,
445
446    #[serde(default)]
447    #[config_doc(nested)]
448    pub meta_store_config: MetaStoreConfig,
449}
450
451#[derive(Copy, Clone, Debug, Default)]
452pub enum DefaultParallelism {
453    #[default]
454    Full,
455    Default(NonZeroUsize),
456}
457
458impl Serialize for DefaultParallelism {
459    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
460    where
461        S: Serializer,
462    {
463        #[derive(Debug, Serialize, Deserialize)]
464        #[serde(untagged)]
465        enum Parallelism {
466            Str(String),
467            Int(usize),
468        }
469        match self {
470            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
471            DefaultParallelism::Default(val) => {
472                Parallelism::Int(val.get() as _).serialize(serializer)
473            }
474        }
475    }
476}
477
478impl<'de> Deserialize<'de> for DefaultParallelism {
479    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
480    where
481        D: serde::Deserializer<'de>,
482    {
483        #[derive(Debug, Deserialize)]
484        #[serde(untagged)]
485        enum Parallelism {
486            Str(String),
487            Int(usize),
488        }
489        let p = Parallelism::deserialize(deserializer)?;
490        match p {
491            Parallelism::Str(s) => {
492                if s.trim().eq_ignore_ascii_case("full") {
493                    Ok(DefaultParallelism::Full)
494                } else {
495                    Err(serde::de::Error::custom(format!(
496                        "invalid default parallelism: {}",
497                        s
498                    )))
499                }
500            }
501            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
502                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
503                // here because it requires extra context. The check will be done when scheduling jobs.
504                NonZeroUsize::new(i).ok_or_else(|| {
505                    serde::de::Error::custom("default parallelism should not be 0")
506                })?,
507            )),
508        }
509    }
510}
511
512/// The subsections `[meta.developer]`.
513///
514/// It is put at [`MetaConfig::developer`].
515#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
516pub struct MetaDeveloperConfig {
517    /// The number of traces to be cached in-memory by the tracing collector
518    /// embedded in the meta node.
519    #[serde(default = "default::developer::meta_cached_traces_num")]
520    pub cached_traces_num: u32,
521
522    /// The maximum memory usage in bytes for the tracing collector embedded
523    /// in the meta node.
524    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
525    pub cached_traces_memory_limit_bytes: usize,
526
527    /// Compaction picker config
528    #[serde(default = "default::developer::enable_trivial_move")]
529    pub enable_trivial_move: bool,
530    #[serde(default = "default::developer::enable_check_task_level_overlap")]
531    pub enable_check_task_level_overlap: bool,
532    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
533    pub max_trivial_move_task_count_per_loop: usize,
534
535    #[serde(default = "default::developer::max_get_task_probe_times")]
536    pub max_get_task_probe_times: usize,
537
538    /// Max number of actor allowed per parallelism (default = 100).
539    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
540    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
541    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
542
543    /// Max number of actor allowed per parallelism (default = 400).
544    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
545    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
546    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
547
548    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
549    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
550    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
551
552    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
553    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
554    pub hummock_time_travel_sst_info_insert_batch_size: usize,
555
556    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
557    pub time_travel_vacuum_interval_sec: u64,
558
559    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
560    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
561    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
562
563    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
564    pub hummock_gc_history_insert_batch_size: usize,
565
566    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
567    pub hummock_time_travel_filter_out_objects_batch_size: usize,
568
569    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
570    pub hummock_time_travel_filter_out_objects_v1: bool,
571
572    #[serde(
573        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
574    )]
575    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
576
577    #[serde(
578        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
579    )]
580    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
581
582    #[serde(default)]
583    pub compute_client_config: RpcClientConfig,
584
585    #[serde(default)]
586    pub stream_client_config: RpcClientConfig,
587
588    #[serde(default)]
589    pub frontend_client_config: RpcClientConfig,
590}
591
592#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
593pub struct RpcClientConfig {
594    #[serde(default = "default::developer::rpc_client_connect_timeout_secs")]
595    pub connect_timeout_secs: u64,
596}
597
598/// The section `[server]` in `risingwave.toml`.
599#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
600pub struct ServerConfig {
601    /// The interval for periodic heartbeat from worker to the meta service.
602    #[serde(default = "default::server::heartbeat_interval_ms")]
603    pub heartbeat_interval_ms: u32,
604
605    /// The default number of the connections when connecting to a gRPC server.
606    ///
607    /// For the connections used in streaming or batch exchange, please refer to the entries in
608    /// `[stream.developer]` and `[batch.developer]` sections. This value will be used if they
609    /// are not specified.
610    #[serde(default = "default::server::connection_pool_size")]
611    // Intentionally made private to avoid abuse. Check the related methods on `RwConfig`.
612    connection_pool_size: u16,
613
614    /// Used for control the metrics level, similar to log level.
615    #[serde(default = "default::server::metrics_level")]
616    pub metrics_level: MetricLevel,
617
618    #[serde(default = "default::server::telemetry_enabled")]
619    pub telemetry_enabled: bool,
620
621    /// Enable heap profile dump when memory usage is high.
622    #[serde(default)]
623    pub heap_profiling: HeapProfilingConfig,
624
625    // Number of max pending reset stream for grpc server.
626    #[serde(default = "default::server::grpc_max_reset_stream_size")]
627    pub grpc_max_reset_stream: u32,
628
629    #[serde(default, flatten)]
630    #[config_doc(omitted)]
631    pub unrecognized: Unrecognized<Self>,
632}
633
634/// The section `[batch]` in `risingwave.toml`.
635#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
636pub struct BatchConfig {
637    /// The thread number of the batch task runtime in the compute node. The default value is
638    /// decided by `tokio`.
639    #[serde(default)]
640    pub worker_threads_num: Option<usize>,
641
642    #[serde(default, with = "batch_prefix")]
643    #[config_doc(omitted)]
644    pub developer: BatchDeveloperConfig,
645
646    /// This is the max number of queries per sql session.
647    #[serde(default)]
648    pub distributed_query_limit: Option<u64>,
649
650    /// This is the max number of batch queries per frontend node.
651    #[serde(default)]
652    pub max_batch_queries_per_frontend_node: Option<u64>,
653
654    #[serde(default = "default::batch::enable_barrier_read")]
655    pub enable_barrier_read: bool,
656
657    /// Timeout for a batch query in seconds.
658    #[serde(default = "default::batch::statement_timeout_in_sec")]
659    pub statement_timeout_in_sec: u32,
660
661    #[serde(default, flatten)]
662    #[config_doc(omitted)]
663    pub unrecognized: Unrecognized<Self>,
664
665    #[serde(default)]
666    /// frontend compute runtime worker threads
667    pub frontend_compute_runtime_worker_threads: Option<usize>,
668
669    /// This is the secs used to mask a worker unavailable temporarily.
670    #[serde(default = "default::batch::mask_worker_temporary_secs")]
671    pub mask_worker_temporary_secs: usize,
672
673    /// Keywords on which SQL option redaction is based in the query log.
674    /// A SQL option with a name containing any of these keywords will be redacted.
675    #[serde(default = "default::batch::redact_sql_option_keywords")]
676    pub redact_sql_option_keywords: Vec<String>,
677
678    /// Enable the spill out to disk feature for batch queries.
679    #[serde(default = "default::batch::enable_spill")]
680    pub enable_spill: bool,
681}
682
683#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
684pub struct FrontendConfig {
685    /// Total memory constraints for running queries.
686    #[serde(default = "default::frontend::max_total_query_size_bytes")]
687    pub max_total_query_size_bytes: u64,
688
689    /// A query of size under this threshold will never be rejected due to memory constraints.
690    #[serde(default = "default::frontend::min_single_query_size_bytes")]
691    pub min_single_query_size_bytes: u64,
692
693    /// A query of size exceeding this threshold will always be rejected due to memory constraints.
694    #[serde(default = "default::frontend::max_single_query_size_bytes")]
695    pub max_single_query_size_bytes: u64,
696}
697
698#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
699pub struct UdfConfig {
700    /// Allow embedded Python UDFs to be created.
701    #[serde(default = "default::udf::enable_embedded_python_udf")]
702    pub enable_embedded_python_udf: bool,
703
704    /// Allow embedded JS UDFs to be created.
705    #[serde(default = "default::udf::enable_embedded_javascript_udf")]
706    pub enable_embedded_javascript_udf: bool,
707
708    /// Allow embedded WASM UDFs to be created.
709    #[serde(default = "default::udf::enable_embedded_wasm_udf")]
710    pub enable_embedded_wasm_udf: bool,
711}
712
713/// The section `[streaming]` in `risingwave.toml`.
714#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
715pub struct StreamingConfig {
716    /// The maximum number of barriers in-flight in the compute nodes.
717    #[serde(default = "default::streaming::in_flight_barrier_nums")]
718    pub in_flight_barrier_nums: usize,
719
720    /// The thread number of the streaming actor runtime in the compute node. The default value is
721    /// decided by `tokio`.
722    #[serde(default)]
723    pub actor_runtime_worker_threads_num: Option<usize>,
724
725    /// Enable async stack tracing through `await-tree` for risectl.
726    #[serde(default = "default::streaming::async_stack_trace")]
727    pub async_stack_trace: AsyncStackTraceOption,
728
729    #[serde(default, with = "streaming_prefix")]
730    #[config_doc(omitted)]
731    pub developer: StreamingDeveloperConfig,
732
733    /// Max unique user stream errors per actor
734    #[serde(default = "default::streaming::unique_user_stream_errors")]
735    pub unique_user_stream_errors: usize,
736
737    /// Control the strictness of stream consistency.
738    #[serde(default = "default::streaming::unsafe_enable_strict_consistency")]
739    pub unsafe_enable_strict_consistency: bool,
740
741    #[serde(default, flatten)]
742    #[config_doc(omitted)]
743    pub unrecognized: Unrecognized<Self>,
744}
745
746pub use risingwave_common_metrics::MetricLevel;
747
748/// the section `[storage.cache]` in `risingwave.toml`.
749#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
750pub struct CacheConfig {
751    /// Configure the capacity of the block cache in MB explicitly.
752    /// The overridden value will only be effective if:
753    /// 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
754    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
755    #[serde(default)]
756    pub block_cache_capacity_mb: Option<usize>,
757
758    /// Configure the number of shards in the block cache explicitly.
759    /// If not set, the shard number will be determined automatically based on cache capacity.
760    #[serde(default)]
761    pub block_cache_shard_num: Option<usize>,
762
763    #[serde(default)]
764    #[config_doc(omitted)]
765    pub block_cache_eviction: CacheEvictionConfig,
766
767    /// Configure the capacity of the block cache in MB explicitly.
768    /// The overridden value will only be effective if:
769    /// 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
770    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
771    #[serde(default)]
772    pub meta_cache_capacity_mb: Option<usize>,
773
774    /// Configure the number of shards in the meta cache explicitly.
775    /// If not set, the shard number will be determined automatically based on cache capacity.
776    #[serde(default)]
777    pub meta_cache_shard_num: Option<usize>,
778
779    #[serde(default)]
780    #[config_doc(omitted)]
781    pub meta_cache_eviction: CacheEvictionConfig,
782
783    #[serde(default = "default::storage::vector_block_cache_capacity_mb")]
784    pub vector_block_cache_capacity_mb: usize,
785    #[serde(default = "default::storage::vector_block_cache_shard_num")]
786    pub vector_block_cache_shard_num: usize,
787    #[serde(default)]
788    #[config_doc(omitted)]
789    pub vector_block_cache_eviction_config: CacheEvictionConfig,
790    #[serde(default = "default::storage::vector_meta_cache_capacity_mb")]
791    pub vector_meta_cache_capacity_mb: usize,
792    #[serde(default = "default::storage::vector_meta_cache_shard_num")]
793    pub vector_meta_cache_shard_num: usize,
794    #[serde(default)]
795    #[config_doc(omitted)]
796    pub vector_meta_cache_eviction_config: CacheEvictionConfig,
797}
798
799/// the section `[storage.cache.eviction]` in `risingwave.toml`.
800#[derive(Clone, Debug, Serialize, Deserialize)]
801#[serde(tag = "algorithm")]
802pub enum CacheEvictionConfig {
803    Lru {
804        high_priority_ratio_in_percent: Option<usize>,
805    },
806    Lfu {
807        window_capacity_ratio_in_percent: Option<usize>,
808        protected_capacity_ratio_in_percent: Option<usize>,
809        cmsketch_eps: Option<f64>,
810        cmsketch_confidence: Option<f64>,
811    },
812    S3Fifo {
813        small_queue_capacity_ratio_in_percent: Option<usize>,
814        ghost_queue_capacity_ratio_in_percent: Option<usize>,
815        small_to_main_freq_threshold: Option<u8>,
816    },
817}
818
819impl Default for CacheEvictionConfig {
820    fn default() -> Self {
821        Self::Lru {
822            high_priority_ratio_in_percent: None,
823        }
824    }
825}
826
827/// The section `[storage]` in `risingwave.toml`.
828#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
829pub struct StorageConfig {
830    /// parallelism while syncing share buffers into L0 SST. Should NOT be 0.
831    #[serde(default = "default::storage::share_buffers_sync_parallelism")]
832    pub share_buffers_sync_parallelism: u32,
833
834    /// Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use
835    /// tokio's default value (number of CPU core).
836    #[serde(default = "default::storage::share_buffer_compaction_worker_threads_number")]
837    pub share_buffer_compaction_worker_threads_number: u32,
838
839    /// Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity
840    /// will stall until there is enough space. The overridden value will only be effective if:
841    /// 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly.
842    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
843    #[serde(default)]
844    pub shared_buffer_capacity_mb: Option<usize>,
845
846    /// The shared buffer will start flushing data to object when the ratio of memory usage to the
847    /// shared buffer capacity exceed such ratio.
848    #[serde(default = "default::storage::shared_buffer_flush_ratio")]
849    pub shared_buffer_flush_ratio: f32,
850
851    /// The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger,
852    /// the total flush size across multiple epochs should be at least higher than this size.
853    #[serde(default = "default::storage::shared_buffer_min_batch_flush_size_mb")]
854    pub shared_buffer_min_batch_flush_size_mb: usize,
855
856    /// The threshold for the number of immutable memtables to merge to a new imm.
857    #[serde(default = "default::storage::imm_merge_threshold")]
858    #[deprecated]
859    pub imm_merge_threshold: usize,
860
861    /// Whether to enable write conflict detection
862    #[serde(default = "default::storage::write_conflict_detection_enabled")]
863    pub write_conflict_detection_enabled: bool,
864
865    #[serde(default)]
866    #[config_doc(nested)]
867    pub cache: CacheConfig,
868
869    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead.
870    #[serde(default)]
871    pub block_cache_capacity_mb: Option<usize>,
872
873    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead.
874    #[serde(default)]
875    pub meta_cache_capacity_mb: Option<usize>,
876
877    /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead.
878    #[serde(default)]
879    pub high_priority_ratio_in_percent: Option<usize>,
880
881    /// max memory usage for large query
882    #[serde(default)]
883    pub prefetch_buffer_capacity_mb: Option<usize>,
884
885    #[serde(default = "default::storage::max_cached_recent_versions_number")]
886    pub max_cached_recent_versions_number: usize,
887
888    /// max prefetch block number
889    #[serde(default = "default::storage::max_prefetch_block_number")]
890    pub max_prefetch_block_number: usize,
891
892    #[serde(default = "default::storage::disable_remote_compactor")]
893    pub disable_remote_compactor: bool,
894
895    /// Number of tasks shared buffer can upload in parallel.
896    #[serde(default = "default::storage::share_buffer_upload_concurrency")]
897    pub share_buffer_upload_concurrency: usize,
898
899    #[serde(default)]
900    pub compactor_memory_limit_mb: Option<usize>,
901
902    /// Compactor calculates the maximum number of tasks that can be executed on the node based on
903    /// `worker_num` and `compactor_max_task_multiplier`.
904    /// `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier`
905    #[serde(default = "default::storage::compactor_max_task_multiplier")]
906    pub compactor_max_task_multiplier: f32,
907
908    /// The percentage of memory available when compactor is deployed separately.
909    /// `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion`
910    #[serde(default = "default::storage::compactor_memory_available_proportion")]
911    pub compactor_memory_available_proportion: f64,
912
913    /// Number of SST ids fetched from meta per RPC
914    #[serde(default = "default::storage::sstable_id_remote_fetch_number")]
915    pub sstable_id_remote_fetch_number: u32,
916
917    #[serde(default = "default::storage::min_sstable_size_mb")]
918    pub min_sstable_size_mb: u32,
919
920    #[serde(default)]
921    #[config_doc(nested)]
922    pub data_file_cache: FileCacheConfig,
923
924    #[serde(default)]
925    #[config_doc(nested)]
926    pub meta_file_cache: FileCacheConfig,
927
928    #[serde(default)]
929    #[config_doc(nested)]
930    pub cache_refill: CacheRefillConfig,
931
932    /// Whether to enable streaming upload for sstable.
933    #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
934    pub min_sst_size_for_streaming_upload: u64,
935
936    #[serde(default = "default::storage::max_concurrent_compaction_task_number")]
937    pub max_concurrent_compaction_task_number: u64,
938
939    #[serde(default = "default::storage::max_preload_wait_time_mill")]
940    pub max_preload_wait_time_mill: u64,
941
942    #[serde(default = "default::storage::max_version_pinning_duration_sec")]
943    pub max_version_pinning_duration_sec: u64,
944
945    #[serde(default = "default::storage::compactor_max_sst_key_count")]
946    pub compactor_max_sst_key_count: u64,
947    // DEPRECATED: This config will be deprecated in the future version, use `storage.compactor_iter_max_io_retry_times` instead.
948    #[serde(default = "default::storage::compact_iter_recreate_timeout_ms")]
949    pub compact_iter_recreate_timeout_ms: u64,
950    #[serde(default = "default::storage::compactor_max_sst_size")]
951    pub compactor_max_sst_size: u64,
952    #[serde(default = "default::storage::enable_fast_compaction")]
953    pub enable_fast_compaction: bool,
954    #[serde(default = "default::storage::check_compaction_result")]
955    pub check_compaction_result: bool,
956    #[serde(default = "default::storage::max_preload_io_retry_times")]
957    pub max_preload_io_retry_times: usize,
958    #[serde(default = "default::storage::compactor_fast_max_compact_delete_ratio")]
959    pub compactor_fast_max_compact_delete_ratio: u32,
960    #[serde(default = "default::storage::compactor_fast_max_compact_task_size")]
961    pub compactor_fast_max_compact_task_size: u64,
962    #[serde(default = "default::storage::compactor_iter_max_io_retry_times")]
963    pub compactor_iter_max_io_retry_times: usize,
964
965    /// Deprecated: The window size of table info statistic history.
966    #[serde(default = "default::storage::table_info_statistic_history_times")]
967    #[deprecated]
968    pub table_info_statistic_history_times: usize,
969
970    #[serde(default, flatten)]
971    #[config_doc(omitted)]
972    pub unrecognized: Unrecognized<Self>,
973
974    /// The spill threshold for mem table.
975    #[serde(default = "default::storage::mem_table_spill_threshold")]
976    pub mem_table_spill_threshold: usize,
977
978    /// The concurrent uploading number of `SSTables` of builder
979    #[serde(default = "default::storage::compactor_concurrent_uploading_sst_count")]
980    pub compactor_concurrent_uploading_sst_count: Option<usize>,
981
982    #[serde(default = "default::storage::compactor_max_overlap_sst_count")]
983    pub compactor_max_overlap_sst_count: usize,
984
985    /// The maximum number of meta files that can be preloaded.
986    /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`.
987    /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient.
988    #[serde(default = "default::storage::compactor_max_preload_meta_file_count")]
989    pub compactor_max_preload_meta_file_count: usize,
990
991    #[serde(default = "default::storage::vector_file_block_size_kb")]
992    pub vector_file_block_size_kb: usize,
993
994    /// Object storage configuration
995    /// 1. General configuration
996    /// 2. Some special configuration of Backend
997    /// 3. Retry and timeout configuration
998    #[serde(default)]
999    pub object_store: ObjectStoreConfig,
1000
1001    #[serde(default = "default::storage::time_travel_version_cache_capacity")]
1002    pub time_travel_version_cache_capacity: u64,
1003
1004    // iceberg compaction
1005    #[serde(default = "default::storage::iceberg_compaction_target_file_size_mb")]
1006    pub iceberg_compaction_target_file_size_mb: u32,
1007    #[serde(default = "default::storage::iceberg_compaction_enable_validate")]
1008    pub iceberg_compaction_enable_validate: bool,
1009    #[serde(default = "default::storage::iceberg_compaction_max_record_batch_rows")]
1010    pub iceberg_compaction_max_record_batch_rows: usize,
1011    #[serde(default = "default::storage::iceberg_compaction_min_size_per_partition_mb")]
1012    pub iceberg_compaction_min_size_per_partition_mb: u32,
1013    #[serde(default = "default::storage::iceberg_compaction_max_file_count_per_partition")]
1014    pub iceberg_compaction_max_file_count_per_partition: u32,
1015
1016    #[serde(default = "default::storage::iceberg_compaction_write_parquet_max_row_group_rows")]
1017    pub iceberg_compaction_write_parquet_max_row_group_rows: usize,
1018}
1019
1020#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1021pub struct CacheRefillConfig {
1022    /// `SSTable` levels to refill.
1023    #[serde(default = "default::cache_refill::data_refill_levels")]
1024    pub data_refill_levels: Vec<u32>,
1025
1026    /// Cache refill maximum timeout to apply version delta.
1027    #[serde(default = "default::cache_refill::timeout_ms")]
1028    pub timeout_ms: u64,
1029
1030    /// Inflight data cache refill tasks.
1031    #[serde(default = "default::cache_refill::concurrency")]
1032    pub concurrency: usize,
1033
1034    /// Block count that a data cache refill request fetches.
1035    #[serde(default = "default::cache_refill::unit")]
1036    pub unit: usize,
1037
1038    /// Data cache refill unit admission ratio.
1039    ///
1040    /// Only unit whose blocks are admitted above the ratio will be refilled.
1041    #[serde(default = "default::cache_refill::threshold")]
1042    pub threshold: f64,
1043
1044    /// Recent filter layer count.
1045    #[serde(default = "default::cache_refill::recent_filter_layers")]
1046    pub recent_filter_layers: usize,
1047
1048    /// Recent filter layer rotate interval.
1049    #[serde(default = "default::cache_refill::recent_filter_rotate_interval_ms")]
1050    pub recent_filter_rotate_interval_ms: usize,
1051
1052    #[serde(default, flatten)]
1053    #[config_doc(omitted)]
1054    pub unrecognized: Unrecognized<Self>,
1055}
1056
1057/// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`.
1058///
1059/// It's put at [`StorageConfig::data_file_cache`] and  [`StorageConfig::meta_file_cache`].
1060#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1061pub struct FileCacheConfig {
1062    #[serde(default = "default::file_cache::dir")]
1063    pub dir: String,
1064
1065    #[serde(default = "default::file_cache::capacity_mb")]
1066    pub capacity_mb: usize,
1067
1068    #[serde(default = "default::file_cache::file_capacity_mb")]
1069    pub file_capacity_mb: usize,
1070
1071    #[serde(default = "default::file_cache::flushers")]
1072    pub flushers: usize,
1073
1074    #[serde(default = "default::file_cache::reclaimers")]
1075    pub reclaimers: usize,
1076
1077    #[serde(default = "default::file_cache::recover_concurrency")]
1078    pub recover_concurrency: usize,
1079
1080    /// Deprecated soon. Please use `throttle` to do I/O throttling instead.
1081    #[serde(default = "default::file_cache::insert_rate_limit_mb")]
1082    pub insert_rate_limit_mb: usize,
1083
1084    #[serde(default = "default::file_cache::indexer_shards")]
1085    pub indexer_shards: usize,
1086
1087    #[serde(default = "default::file_cache::compression")]
1088    pub compression: Compression,
1089
1090    #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
1091    pub flush_buffer_threshold_mb: Option<usize>,
1092
1093    #[serde(default = "default::file_cache::throttle")]
1094    pub throttle: Throttle,
1095
1096    #[serde(default = "default::file_cache::fifo_probation_ratio")]
1097    pub fifo_probation_ratio: f64,
1098
1099    /// Recover mode.
1100    ///
1101    /// Options:
1102    ///
1103    /// - "None": Do not recover disk cache.
1104    /// - "Quiet": Recover disk cache and skip errors.
1105    /// - "Strict": Recover disk cache and panic on errors.
1106    ///
1107    /// More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`],
1108    #[serde(default = "default::file_cache::recover_mode")]
1109    pub recover_mode: RecoverMode,
1110
1111    #[serde(default = "default::file_cache::runtime_config")]
1112    pub runtime_config: RuntimeOptions,
1113
1114    #[serde(default, flatten)]
1115    #[config_doc(omitted)]
1116    pub unrecognized: Unrecognized<Self>,
1117}
1118
1119#[derive(Debug, Default, Clone, Copy, ValueEnum, Serialize, Deserialize)]
1120pub enum AsyncStackTraceOption {
1121    /// Disabled.
1122    Off,
1123    /// Enabled with basic instruments.
1124    On,
1125    /// Enabled with extra verbose instruments in release build.
1126    /// Behaves the same as `on` in debug build due to performance concern.
1127    #[default]
1128    #[clap(alias = "verbose")]
1129    ReleaseVerbose,
1130}
1131
1132impl AsyncStackTraceOption {
1133    pub fn is_verbose(self) -> Option<bool> {
1134        match self {
1135            Self::Off => None,
1136            Self::On => Some(false),
1137            Self::ReleaseVerbose => Some(!cfg!(debug_assertions)),
1138        }
1139    }
1140}
1141
1142#[derive(Debug, Default, Clone, Copy, ValueEnum)]
1143pub enum CompactorMode {
1144    #[default]
1145    #[clap(alias = "dedicated")]
1146    Dedicated,
1147
1148    #[clap(alias = "shared")]
1149    Shared,
1150
1151    #[clap(alias = "dedicated_iceberg")]
1152    DedicatedIceberg,
1153
1154    #[clap(alias = "shared_iceberg")]
1155    SharedIceberg,
1156}
1157
1158#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1159pub struct HeapProfilingConfig {
1160    /// Enable to auto dump heap profile when memory usage is high
1161    #[serde(default = "default::heap_profiling::enable_auto")]
1162    pub enable_auto: bool,
1163
1164    /// The proportion (number between 0 and 1) of memory usage to trigger heap profile dump
1165    #[serde(default = "default::heap_profiling::threshold_auto")]
1166    pub threshold_auto: f32,
1167
1168    /// The directory to dump heap profile. If empty, the prefix in `MALLOC_CONF` will be used
1169    #[serde(default = "default::heap_profiling::dir")]
1170    pub dir: String,
1171}
1172
1173/// The subsections `[streaming.developer]`.
1174///
1175/// It is put at [`StreamingConfig::developer`].
1176#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1177pub struct StreamingDeveloperConfig {
1178    /// Set to true to enable per-executor row count metrics. This will produce a lot of timeseries
1179    /// and might affect the prometheus performance. If you only need actor input and output
1180    /// rows data, see `stream_actor_in_record_cnt` and `stream_actor_out_record_cnt` instead.
1181    #[serde(default = "default::developer::stream_enable_executor_row_count")]
1182    pub enable_executor_row_count: bool,
1183
1184    /// The capacity of the chunks in the channel that connects between `ConnectorSource` and
1185    /// `SourceExecutor`.
1186    #[serde(default = "default::developer::connector_message_buffer_size")]
1187    pub connector_message_buffer_size: usize,
1188
1189    /// Limit number of the cached entries in an extreme aggregation call.
1190    #[serde(default = "default::developer::unsafe_stream_extreme_cache_size")]
1191    pub unsafe_extreme_cache_size: usize,
1192
1193    /// The maximum size of the chunk produced by executor at a time.
1194    #[serde(default = "default::developer::stream_chunk_size")]
1195    pub chunk_size: usize,
1196
1197    /// The initial permits that a channel holds, i.e., the maximum row count can be buffered in
1198    /// the channel.
1199    #[serde(default = "default::developer::stream_exchange_initial_permits")]
1200    pub exchange_initial_permits: usize,
1201
1202    /// The permits that are batched to add back, for reducing the backward `AddPermits` messages
1203    /// in remote exchange.
1204    #[serde(default = "default::developer::stream_exchange_batched_permits")]
1205    pub exchange_batched_permits: usize,
1206
1207    /// The maximum number of concurrent barriers in an exchange channel.
1208    #[serde(default = "default::developer::stream_exchange_concurrent_barriers")]
1209    pub exchange_concurrent_barriers: usize,
1210
1211    /// The concurrency for dispatching messages to different downstream jobs.
1212    ///
1213    /// - `1` means no concurrency, i.e., dispatch messages to downstream jobs one by one.
1214    /// - `0` means unlimited concurrency.
1215    #[serde(default = "default::developer::stream_exchange_concurrent_dispatchers")]
1216    pub exchange_concurrent_dispatchers: usize,
1217
1218    /// The initial permits for a dml channel, i.e., the maximum row count can be buffered in
1219    /// the channel.
1220    #[serde(default = "default::developer::stream_dml_channel_initial_permits")]
1221    pub dml_channel_initial_permits: usize,
1222
1223    /// The max heap size of dirty groups of `HashAggExecutor`.
1224    #[serde(default = "default::developer::stream_hash_agg_max_dirty_groups_heap_size")]
1225    pub hash_agg_max_dirty_groups_heap_size: usize,
1226
1227    #[serde(default = "default::developer::memory_controller_threshold_aggressive")]
1228    pub memory_controller_threshold_aggressive: f64,
1229
1230    #[serde(default = "default::developer::memory_controller_threshold_graceful")]
1231    pub memory_controller_threshold_graceful: f64,
1232
1233    #[serde(default = "default::developer::memory_controller_threshold_stable")]
1234    pub memory_controller_threshold_stable: f64,
1235
1236    #[serde(default = "default::developer::memory_controller_eviction_factor_aggressive")]
1237    pub memory_controller_eviction_factor_aggressive: f64,
1238
1239    #[serde(default = "default::developer::memory_controller_eviction_factor_graceful")]
1240    pub memory_controller_eviction_factor_graceful: f64,
1241
1242    #[serde(default = "default::developer::memory_controller_eviction_factor_stable")]
1243    pub memory_controller_eviction_factor_stable: f64,
1244
1245    #[serde(default = "default::developer::memory_controller_update_interval_ms")]
1246    pub memory_controller_update_interval_ms: usize,
1247
1248    #[serde(default = "default::developer::memory_controller_sequence_tls_step")]
1249    pub memory_controller_sequence_tls_step: u64,
1250
1251    #[serde(default = "default::developer::memory_controller_sequence_tls_lag")]
1252    pub memory_controller_sequence_tls_lag: u64,
1253
1254    #[serde(default = "default::developer::stream_enable_arrangement_backfill")]
1255    /// Enable arrangement backfill
1256    /// If false, the arrangement backfill will be disabled,
1257    /// even if session variable set.
1258    /// If true, it's decided by session variable `streaming_use_arrangement_backfill` (default true)
1259    pub enable_arrangement_backfill: bool,
1260
1261    #[serde(default = "default::developer::stream_high_join_amplification_threshold")]
1262    /// If number of hash join matches exceeds this threshold number,
1263    /// it will be logged.
1264    pub high_join_amplification_threshold: usize,
1265
1266    /// Actor tokio metrics is enabled if `enable_actor_tokio_metrics` is set or metrics level >= Debug.
1267    #[serde(default = "default::developer::enable_actor_tokio_metrics")]
1268    pub enable_actor_tokio_metrics: bool,
1269
1270    /// The number of the connections for streaming remote exchange between two nodes.
1271    /// If not specified, the value of `server.connection_pool_size` will be used.
1272    #[serde(default = "default::developer::stream_exchange_connection_pool_size")]
1273    pub exchange_connection_pool_size: Option<u16>,
1274
1275    /// A flag to allow disabling the auto schema change handling
1276    #[serde(default = "default::developer::stream_enable_auto_schema_change")]
1277    pub enable_auto_schema_change: bool,
1278
1279    #[serde(default = "default::developer::enable_shared_source")]
1280    /// Enable shared source
1281    /// If false, the shared source will be disabled,
1282    /// even if session variable set.
1283    /// If true, it's decided by session variable `streaming_use_shared_source` (default true)
1284    pub enable_shared_source: bool,
1285
1286    #[serde(default = "default::developer::switch_jdbc_pg_to_native")]
1287    /// When true, all jdbc sinks with connector='jdbc' and jdbc.url="jdbc:postgresql://..."
1288    /// will be switched from jdbc postgresql sinks to rust native (connector='postgres') sinks.
1289    pub switch_jdbc_pg_to_native: bool,
1290
1291    /// The maximum number of consecutive barriers allowed in a message when sent between actors.
1292    #[serde(default = "default::developer::stream_max_barrier_batch_size")]
1293    pub max_barrier_batch_size: u32,
1294
1295    /// Configure the system-wide cache row cardinality of hash join.
1296    /// For example, if this is set to 1000, it means we can have at most 1000 rows in cache.
1297    #[serde(default = "default::developer::streaming_hash_join_entry_state_max_rows")]
1298    pub hash_join_entry_state_max_rows: usize,
1299
1300    /// Enable / Disable profiling stats used by `EXPLAIN ANALYZE`
1301    #[serde(default = "default::developer::enable_explain_analyze_stats")]
1302    pub enable_explain_analyze_stats: bool,
1303
1304    #[serde(default)]
1305    pub compute_client_config: RpcClientConfig,
1306
1307    /// `IcebergListExecutor`: The interval in seconds for Iceberg source to list new files.
1308    #[serde(default = "default::developer::iceberg_list_interval_sec")]
1309    pub iceberg_list_interval_sec: u64,
1310
1311    /// `IcebergFetchExecutor`: The number of files the executor will fetch concurrently in a batch.
1312    #[serde(default = "default::developer::iceberg_fetch_batch_size")]
1313    pub iceberg_fetch_batch_size: u64,
1314
1315    /// `IcebergSink`: The size of the cache for positional delete in the sink.
1316    #[serde(default = "default::developer::iceberg_sink_positional_delete_cache_size")]
1317    pub iceberg_sink_positional_delete_cache_size: usize,
1318}
1319
1320/// The subsections `[batch.developer]`.
1321///
1322/// It is put at [`BatchConfig::developer`].
1323#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1324pub struct BatchDeveloperConfig {
1325    /// The capacity of the chunks in the channel that connects between `ConnectorSource` and
1326    /// `SourceExecutor`.
1327    #[serde(default = "default::developer::connector_message_buffer_size")]
1328    pub connector_message_buffer_size: usize,
1329
1330    /// The size of the channel used for output to exchange/shuffle.
1331    #[serde(default = "default::developer::batch_output_channel_size")]
1332    pub output_channel_size: usize,
1333
1334    #[serde(default = "default::developer::batch_receiver_channel_size")]
1335    pub receiver_channel_size: usize,
1336
1337    #[serde(default = "default::developer::batch_root_stage_channel_size")]
1338    pub root_stage_channel_size: usize,
1339
1340    /// The size of a chunk produced by `RowSeqScanExecutor`
1341    #[serde(default = "default::developer::batch_chunk_size")]
1342    pub chunk_size: usize,
1343
1344    /// The number of the connections for batch remote exchange between two nodes.
1345    /// If not specified, the value of `server.connection_pool_size` will be used.
1346    #[serde(default = "default::developer::batch_exchange_connection_pool_size")]
1347    exchange_connection_pool_size: Option<u16>,
1348
1349    #[serde(default)]
1350    pub compute_client_config: RpcClientConfig,
1351
1352    #[serde(default)]
1353    pub frontend_client_config: RpcClientConfig,
1354
1355    #[serde(default = "default::developer::batch_local_execute_buffer_size")]
1356    pub local_execute_buffer_size: usize,
1357}
1358
1359macro_rules! define_system_config {
1360    ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $doc:literal, $($rest:tt)* },)*) => {
1361        paste::paste!(
1362            /// The section `[system]` in `risingwave.toml`. All these fields are used to initialize the system
1363            /// parameters persisted in Meta store. Most fields are for testing purpose only and should not be
1364            /// documented.
1365            #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
1366            pub struct SystemConfig {
1367                $(
1368                    #[doc = $doc]
1369                    #[serde(default = "default::system::" $field "_opt")]
1370                    pub $field: Option<$type>,
1371                )*
1372            }
1373        );
1374    };
1375}
1376
1377for_all_params!(define_system_config);
1378
1379/// The subsections `[storage.object_store]`.
1380#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
1381pub struct ObjectStoreConfig {
1382    // alias is for backward compatibility
1383    #[serde(
1384        default = "default::object_store_config::set_atomic_write_dir",
1385        alias = "object_store_set_atomic_write_dir"
1386    )]
1387    pub set_atomic_write_dir: bool,
1388
1389    /// Retry and timeout configuration
1390    /// Description retry strategy driven by exponential back-off
1391    /// Exposes the timeout and retries of each Object store interface. Therefore, the total timeout for each interface is determined based on the interface's timeout/retry configuration and the exponential back-off policy.
1392    #[serde(default)]
1393    pub retry: ObjectStoreRetryConfig,
1394
1395    /// Some special configuration of S3 Backend
1396    #[serde(default)]
1397    pub s3: S3ObjectStoreConfig,
1398
1399    // TODO: the following field will be deprecated after opendal is stabilized
1400    #[serde(default = "default::object_store_config::opendal_upload_concurrency")]
1401    pub opendal_upload_concurrency: usize,
1402
1403    // TODO: the following field will be deprecated after opendal is stabilized
1404    #[serde(default)]
1405    pub opendal_writer_abort_on_err: bool,
1406
1407    #[serde(default = "default::object_store_config::upload_part_size")]
1408    pub upload_part_size: usize,
1409}
1410
1411impl ObjectStoreConfig {
1412    pub fn set_atomic_write_dir(&mut self) {
1413        self.set_atomic_write_dir = true;
1414    }
1415}
1416
1417/// The subsections `[storage.object_store.s3]`.
1418#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
1419pub struct S3ObjectStoreConfig {
1420    // alias is for backward compatibility
1421    #[serde(
1422        default = "default::object_store_config::s3::keepalive_ms",
1423        alias = "object_store_keepalive_ms"
1424    )]
1425    pub keepalive_ms: Option<u64>,
1426    #[serde(
1427        default = "default::object_store_config::s3::recv_buffer_size",
1428        alias = "object_store_recv_buffer_size"
1429    )]
1430    pub recv_buffer_size: Option<usize>,
1431    #[serde(
1432        default = "default::object_store_config::s3::send_buffer_size",
1433        alias = "object_store_send_buffer_size"
1434    )]
1435    pub send_buffer_size: Option<usize>,
1436    #[serde(
1437        default = "default::object_store_config::s3::nodelay",
1438        alias = "object_store_nodelay"
1439    )]
1440    pub nodelay: Option<bool>,
1441    /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead.
1442    #[serde(default = "default::object_store_config::s3::developer::retry_unknown_service_error")]
1443    pub retry_unknown_service_error: bool,
1444    #[serde(default = "default::object_store_config::s3::identity_resolution_timeout_s")]
1445    pub identity_resolution_timeout_s: u64,
1446    #[serde(default)]
1447    pub developer: S3ObjectStoreDeveloperConfig,
1448}
1449
1450/// The subsections `[storage.object_store.s3.developer]`.
1451#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
1452pub struct S3ObjectStoreDeveloperConfig {
1453    /// Whether to retry s3 sdk error from which no error metadata is provided.
1454    #[serde(
1455        default = "default::object_store_config::s3::developer::retry_unknown_service_error",
1456        alias = "object_store_retry_unknown_service_error"
1457    )]
1458    pub retry_unknown_service_error: bool,
1459    /// An array of error codes that should be retried.
1460    /// e.g. `["SlowDown", "TooManyRequests"]`
1461    #[serde(
1462        default = "default::object_store_config::s3::developer::retryable_service_error_codes",
1463        alias = "object_store_retryable_service_error_codes"
1464    )]
1465    pub retryable_service_error_codes: Vec<String>,
1466
1467    // TODO: deprecate this config when we are completely deprecate aws sdk.
1468    #[serde(default = "default::object_store_config::s3::developer::use_opendal")]
1469    pub use_opendal: bool,
1470}
1471
1472#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
1473pub struct ObjectStoreRetryConfig {
1474    // A retry strategy driven by exponential back-off.
1475    // The retry strategy is used for all object store operations.
1476    /// Given a base duration for retry strategy in milliseconds.
1477    #[serde(default = "default::object_store_config::object_store_req_backoff_interval_ms")]
1478    pub req_backoff_interval_ms: u64,
1479
1480    /// The max delay interval for the retry strategy. No retry delay will be longer than this `Duration`.
1481    #[serde(default = "default::object_store_config::object_store_req_backoff_max_delay_ms")]
1482    pub req_backoff_max_delay_ms: u64,
1483
1484    /// A multiplicative factor that will be applied to the exponential back-off retry delay.
1485    #[serde(default = "default::object_store_config::object_store_req_backoff_factor")]
1486    pub req_backoff_factor: u64,
1487
1488    /// Maximum timeout for `upload` operation
1489    #[serde(default = "default::object_store_config::object_store_upload_attempt_timeout_ms")]
1490    pub upload_attempt_timeout_ms: u64,
1491
1492    /// Total counts of `upload` operation retries
1493    #[serde(default = "default::object_store_config::object_store_upload_retry_attempts")]
1494    pub upload_retry_attempts: usize,
1495
1496    /// Maximum timeout for `streaming_upload_init` and `streaming_upload`
1497    #[serde(
1498        default = "default::object_store_config::object_store_streaming_upload_attempt_timeout_ms"
1499    )]
1500    pub streaming_upload_attempt_timeout_ms: u64,
1501
1502    /// Total counts of `streaming_upload` operation retries
1503    #[serde(
1504        default = "default::object_store_config::object_store_streaming_upload_retry_attempts"
1505    )]
1506    pub streaming_upload_retry_attempts: usize,
1507
1508    /// Maximum timeout for `read` operation
1509    #[serde(default = "default::object_store_config::object_store_read_attempt_timeout_ms")]
1510    pub read_attempt_timeout_ms: u64,
1511
1512    /// Total counts of `read` operation retries
1513    #[serde(default = "default::object_store_config::object_store_read_retry_attempts")]
1514    pub read_retry_attempts: usize,
1515
1516    /// Maximum timeout for `streaming_read_init` and `streaming_read` operation
1517    #[serde(
1518        default = "default::object_store_config::object_store_streaming_read_attempt_timeout_ms"
1519    )]
1520    pub streaming_read_attempt_timeout_ms: u64,
1521
1522    /// Total counts of `streaming_read operation` retries
1523    #[serde(default = "default::object_store_config::object_store_streaming_read_retry_attempts")]
1524    pub streaming_read_retry_attempts: usize,
1525
1526    /// Maximum timeout for `metadata` operation
1527    #[serde(default = "default::object_store_config::object_store_metadata_attempt_timeout_ms")]
1528    pub metadata_attempt_timeout_ms: u64,
1529
1530    /// Total counts of `metadata` operation retries
1531    #[serde(default = "default::object_store_config::object_store_metadata_retry_attempts")]
1532    pub metadata_retry_attempts: usize,
1533
1534    /// Maximum timeout for `delete` operation
1535    #[serde(default = "default::object_store_config::object_store_delete_attempt_timeout_ms")]
1536    pub delete_attempt_timeout_ms: u64,
1537
1538    /// Total counts of `delete` operation retries
1539    #[serde(default = "default::object_store_config::object_store_delete_retry_attempts")]
1540    pub delete_retry_attempts: usize,
1541
1542    /// Maximum timeout for `delete_object` operation
1543    #[serde(
1544        default = "default::object_store_config::object_store_delete_objects_attempt_timeout_ms"
1545    )]
1546    pub delete_objects_attempt_timeout_ms: u64,
1547
1548    /// Total counts of `delete_object` operation retries
1549    #[serde(default = "default::object_store_config::object_store_delete_objects_retry_attempts")]
1550    pub delete_objects_retry_attempts: usize,
1551
1552    /// Maximum timeout for `list` operation
1553    #[serde(default = "default::object_store_config::object_store_list_attempt_timeout_ms")]
1554    pub list_attempt_timeout_ms: u64,
1555
1556    /// Total counts of `list` operation retries
1557    #[serde(default = "default::object_store_config::object_store_list_retry_attempts")]
1558    pub list_retry_attempts: usize,
1559}
1560
1561impl SystemConfig {
1562    #![allow(deprecated)]
1563    pub fn into_init_system_params(self) -> SystemParams {
1564        macro_rules! fields {
1565            ($({ $field:ident, $($rest:tt)* },)*) => {
1566                SystemParams {
1567                    $($field: self.$field.map(Into::into),)*
1568                    ..Default::default() // deprecated fields
1569                }
1570            };
1571        }
1572
1573        let mut system_params = for_all_params!(fields);
1574
1575        // Initialize backup_storage_url and backup_storage_directory if not set.
1576        if let Some(state_store) = &system_params.state_store
1577            && let Some(data_directory) = &system_params.data_directory
1578        {
1579            if system_params.backup_storage_url.is_none() {
1580                if let Some(hummock_state_store) = state_store.strip_prefix("hummock+") {
1581                    system_params.backup_storage_url = Some(hummock_state_store.to_owned());
1582                } else {
1583                    system_params.backup_storage_url = Some("memory".to_owned());
1584                }
1585                tracing::info!("initialize backup_storage_url based on state_store");
1586            }
1587            if system_params.backup_storage_directory.is_none() {
1588                system_params.backup_storage_directory = Some(format!("{data_directory}/backup"));
1589                tracing::info!("initialize backup_storage_directory based on data_directory");
1590            }
1591        }
1592        system_params
1593    }
1594}
1595
1596impl RwConfig {
1597    pub const fn default_connection_pool_size(&self) -> u16 {
1598        self.server.connection_pool_size
1599    }
1600
1601    /// Returns [`StreamingDeveloperConfig::exchange_connection_pool_size`] if set,
1602    /// otherwise [`ServerConfig::connection_pool_size`].
1603    pub fn streaming_exchange_connection_pool_size(&self) -> u16 {
1604        self.streaming
1605            .developer
1606            .exchange_connection_pool_size
1607            .unwrap_or_else(|| self.default_connection_pool_size())
1608    }
1609
1610    /// Returns [`BatchDeveloperConfig::exchange_connection_pool_size`] if set,
1611    /// otherwise [`ServerConfig::connection_pool_size`].
1612    pub fn batch_exchange_connection_pool_size(&self) -> u16 {
1613        self.batch
1614            .developer
1615            .exchange_connection_pool_size
1616            .unwrap_or_else(|| self.default_connection_pool_size())
1617    }
1618}
1619
1620pub mod default {
1621    pub mod meta {
1622        use crate::config::{DefaultParallelism, MetaBackend};
1623
1624        pub fn min_sst_retention_time_sec() -> u64 {
1625            3600 * 6
1626        }
1627
1628        pub fn gc_history_retention_time_sec() -> u64 {
1629            3600 * 6
1630        }
1631
1632        pub fn full_gc_interval_sec() -> u64 {
1633            3600
1634        }
1635
1636        pub fn full_gc_object_limit() -> u64 {
1637            100_000
1638        }
1639
1640        pub fn max_inflight_time_travel_query() -> u64 {
1641            1000
1642        }
1643
1644        pub fn periodic_compaction_interval_sec() -> u64 {
1645            60
1646        }
1647
1648        pub fn vacuum_interval_sec() -> u64 {
1649            30
1650        }
1651
1652        pub fn vacuum_spin_interval_ms() -> u64 {
1653            100
1654        }
1655
1656        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
1657            30
1658        }
1659
1660        pub fn enable_hummock_data_archive() -> bool {
1661            false
1662        }
1663
1664        pub fn hummock_time_travel_snapshot_interval() -> u64 {
1665            100
1666        }
1667
1668        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
1669            10
1670        }
1671
1672        pub fn max_heartbeat_interval_sec() -> u32 {
1673            60
1674        }
1675
1676        pub fn meta_leader_lease_secs() -> u64 {
1677            30
1678        }
1679
1680        pub fn default_parallelism() -> DefaultParallelism {
1681            DefaultParallelism::Full
1682        }
1683
1684        pub fn node_num_monitor_interval_sec() -> u64 {
1685            10
1686        }
1687
1688        pub fn backend() -> MetaBackend {
1689            MetaBackend::Mem
1690        }
1691
1692        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
1693            3600 // 60min
1694        }
1695
1696        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
1697            1800 // 30mi
1698        }
1699
1700        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
1701            10 // 10s
1702        }
1703
1704        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
1705            600
1706        }
1707
1708        // limit the size of state table to trigger split by high throughput
1709        pub fn move_table_size_limit() -> u64 {
1710            10 * 1024 * 1024 * 1024 // 10GB
1711        }
1712
1713        // limit the size of group to trigger split by group_size and avoid too many small groups
1714        pub fn split_group_size_limit() -> u64 {
1715            64 * 1024 * 1024 * 1024 // 64GB
1716        }
1717
1718        pub fn protect_drop_table_with_incoming_sink() -> bool {
1719            false
1720        }
1721
1722        pub fn partition_vnode_count() -> u32 {
1723            16
1724        }
1725
1726        pub fn table_high_write_throughput_threshold() -> u64 {
1727            16 * 1024 * 1024 // 16MB
1728        }
1729
1730        pub fn table_low_write_throughput_threshold() -> u64 {
1731            4 * 1024 * 1024 // 4MB
1732        }
1733
1734        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
1735            30 // 30s
1736        }
1737
1738        pub fn compaction_task_max_progress_interval_secs() -> u64 {
1739            60 * 10 // 10min
1740        }
1741
1742        pub fn cut_table_size_limit() -> u64 {
1743            1024 * 1024 * 1024 // 1GB
1744        }
1745
1746        pub fn hybrid_partition_vnode_count() -> u32 {
1747            4
1748        }
1749
1750        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
1751            128 * 1024 * 1024 // 128MB
1752        }
1753
1754        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
1755            512 * 1024 * 1024 // 512MB
1756        }
1757
1758        pub fn event_log_enabled() -> bool {
1759            true
1760        }
1761
1762        pub fn event_log_channel_max_size() -> u32 {
1763            10
1764        }
1765
1766        pub fn parallelism_control_batch_size() -> usize {
1767            10
1768        }
1769
1770        pub fn parallelism_control_trigger_period_sec() -> u64 {
1771            10
1772        }
1773
1774        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
1775            30
1776        }
1777
1778        pub fn enable_dropped_column_reclaim() -> bool {
1779            false
1780        }
1781
1782        pub fn split_group_size_ratio() -> f64 {
1783            0.9
1784        }
1785
1786        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
1787            0.5
1788        }
1789
1790        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
1791            0.7
1792        }
1793
1794        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
1795            60
1796        }
1797
1798        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
1799            240
1800        }
1801
1802        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
1803            60 * 10 // 10min
1804        }
1805
1806        pub fn compaction_group_merge_dimension_threshold() -> f64 {
1807            1.2
1808        }
1809    }
1810
1811    pub mod server {
1812        use crate::config::MetricLevel;
1813
1814        pub fn heartbeat_interval_ms() -> u32 {
1815            1000
1816        }
1817
1818        pub fn connection_pool_size() -> u16 {
1819            16
1820        }
1821
1822        pub fn metrics_level() -> MetricLevel {
1823            MetricLevel::Info
1824        }
1825
1826        pub fn telemetry_enabled() -> bool {
1827            true
1828        }
1829
1830        pub fn grpc_max_reset_stream_size() -> u32 {
1831            200
1832        }
1833    }
1834
1835    pub mod storage {
1836        pub fn share_buffers_sync_parallelism() -> u32 {
1837            1
1838        }
1839
1840        pub fn share_buffer_compaction_worker_threads_number() -> u32 {
1841            4
1842        }
1843
1844        pub fn shared_buffer_capacity_mb() -> usize {
1845            1024
1846        }
1847
1848        pub fn shared_buffer_flush_ratio() -> f32 {
1849            0.8
1850        }
1851
1852        pub fn shared_buffer_min_batch_flush_size_mb() -> usize {
1853            800
1854        }
1855
1856        pub fn imm_merge_threshold() -> usize {
1857            0 // disable
1858        }
1859
1860        pub fn write_conflict_detection_enabled() -> bool {
1861            cfg!(debug_assertions)
1862        }
1863
1864        pub fn max_cached_recent_versions_number() -> usize {
1865            60
1866        }
1867
1868        pub fn block_cache_capacity_mb() -> usize {
1869            512
1870        }
1871
1872        pub fn high_priority_ratio_in_percent() -> usize {
1873            70
1874        }
1875
1876        pub fn window_capacity_ratio_in_percent() -> usize {
1877            10
1878        }
1879
1880        pub fn protected_capacity_ratio_in_percent() -> usize {
1881            80
1882        }
1883
1884        pub fn cmsketch_eps() -> f64 {
1885            0.002
1886        }
1887
1888        pub fn cmsketch_confidence() -> f64 {
1889            0.95
1890        }
1891
1892        pub fn small_queue_capacity_ratio_in_percent() -> usize {
1893            10
1894        }
1895
1896        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
1897            1000
1898        }
1899
1900        pub fn small_to_main_freq_threshold() -> u8 {
1901            1
1902        }
1903
1904        pub fn meta_cache_capacity_mb() -> usize {
1905            128
1906        }
1907
1908        pub fn disable_remote_compactor() -> bool {
1909            false
1910        }
1911
1912        pub fn share_buffer_upload_concurrency() -> usize {
1913            8
1914        }
1915
1916        pub fn compactor_memory_limit_mb() -> usize {
1917            512
1918        }
1919
1920        pub fn compactor_max_task_multiplier() -> f32 {
1921            3.0000
1922        }
1923
1924        pub fn compactor_memory_available_proportion() -> f64 {
1925            0.8
1926        }
1927
1928        pub fn sstable_id_remote_fetch_number() -> u32 {
1929            10
1930        }
1931
1932        pub fn min_sstable_size_mb() -> u32 {
1933            32
1934        }
1935
1936        pub fn min_sst_size_for_streaming_upload() -> u64 {
1937            // 32MB
1938            32 * 1024 * 1024
1939        }
1940
1941        pub fn max_concurrent_compaction_task_number() -> u64 {
1942            16
1943        }
1944
1945        pub fn max_preload_wait_time_mill() -> u64 {
1946            0
1947        }
1948
1949        pub fn max_version_pinning_duration_sec() -> u64 {
1950            3 * 3600
1951        }
1952
1953        pub fn compactor_max_sst_key_count() -> u64 {
1954            2 * 1024 * 1024 // 200w
1955        }
1956
1957        pub fn compact_iter_recreate_timeout_ms() -> u64 {
1958            10 * 60 * 1000
1959        }
1960
1961        pub fn compactor_iter_max_io_retry_times() -> usize {
1962            8
1963        }
1964
1965        pub fn compactor_max_sst_size() -> u64 {
1966            512 * 1024 * 1024 // 512m
1967        }
1968
1969        pub fn enable_fast_compaction() -> bool {
1970            true
1971        }
1972
1973        pub fn check_compaction_result() -> bool {
1974            false
1975        }
1976
1977        pub fn max_preload_io_retry_times() -> usize {
1978            3
1979        }
1980
1981        pub fn mem_table_spill_threshold() -> usize {
1982            4 << 20
1983        }
1984
1985        pub fn compactor_fast_max_compact_delete_ratio() -> u32 {
1986            40
1987        }
1988
1989        pub fn compactor_fast_max_compact_task_size() -> u64 {
1990            2 * 1024 * 1024 * 1024 // 2g
1991        }
1992
1993        pub fn max_prefetch_block_number() -> usize {
1994            16
1995        }
1996
1997        pub fn compactor_concurrent_uploading_sst_count() -> Option<usize> {
1998            None
1999        }
2000
2001        pub fn compactor_max_overlap_sst_count() -> usize {
2002            64
2003        }
2004
2005        pub fn compactor_max_preload_meta_file_count() -> usize {
2006            32
2007        }
2008
2009        pub fn vector_file_block_size_kb() -> usize {
2010            1024
2011        }
2012
2013        pub fn vector_block_cache_capacity_mb() -> usize {
2014            16
2015        }
2016
2017        pub fn vector_block_cache_shard_num() -> usize {
2018            16
2019        }
2020
2021        pub fn vector_meta_cache_capacity_mb() -> usize {
2022            16
2023        }
2024
2025        pub fn vector_meta_cache_shard_num() -> usize {
2026            16
2027        }
2028
2029        // deprecated
2030        pub fn table_info_statistic_history_times() -> usize {
2031            240
2032        }
2033
2034        pub fn block_file_cache_flush_buffer_threshold_mb() -> usize {
2035            256
2036        }
2037
2038        pub fn meta_file_cache_flush_buffer_threshold_mb() -> usize {
2039            64
2040        }
2041
2042        pub fn time_travel_version_cache_capacity() -> u64 {
2043            10
2044        }
2045
2046        pub fn iceberg_compaction_target_file_size_mb() -> u32 {
2047            1024
2048        }
2049
2050        pub fn iceberg_compaction_enable_validate() -> bool {
2051            false
2052        }
2053
2054        pub fn iceberg_compaction_max_record_batch_rows() -> usize {
2055            1024
2056        }
2057
2058        pub fn iceberg_compaction_write_parquet_max_row_group_rows() -> usize {
2059            1024 * 100 // 100k
2060        }
2061
2062        pub fn iceberg_compaction_min_size_per_partition_mb() -> u32 {
2063            1024
2064        }
2065
2066        pub fn iceberg_compaction_max_file_count_per_partition() -> u32 {
2067            32
2068        }
2069    }
2070
2071    pub mod streaming {
2072        use crate::config::AsyncStackTraceOption;
2073
2074        pub fn in_flight_barrier_nums() -> usize {
2075            // quick fix
2076            // TODO: remove this limitation from code
2077            10000
2078        }
2079
2080        pub fn async_stack_trace() -> AsyncStackTraceOption {
2081            AsyncStackTraceOption::default()
2082        }
2083
2084        pub fn unique_user_stream_errors() -> usize {
2085            10
2086        }
2087
2088        pub fn unsafe_enable_strict_consistency() -> bool {
2089            true
2090        }
2091    }
2092
2093    pub mod file_cache {
2094        use std::num::NonZeroUsize;
2095
2096        use foyer::{Compression, RecoverMode, RuntimeOptions, Throttle, TokioRuntimeOptions};
2097
2098        pub fn dir() -> String {
2099            "".to_owned()
2100        }
2101
2102        pub fn capacity_mb() -> usize {
2103            1024
2104        }
2105
2106        pub fn file_capacity_mb() -> usize {
2107            64
2108        }
2109
2110        pub fn flushers() -> usize {
2111            4
2112        }
2113
2114        pub fn reclaimers() -> usize {
2115            4
2116        }
2117
2118        pub fn recover_concurrency() -> usize {
2119            8
2120        }
2121
2122        pub fn insert_rate_limit_mb() -> usize {
2123            0
2124        }
2125
2126        pub fn indexer_shards() -> usize {
2127            64
2128        }
2129
2130        pub fn compression() -> Compression {
2131            Compression::None
2132        }
2133
2134        pub fn flush_buffer_threshold_mb() -> Option<usize> {
2135            None
2136        }
2137
2138        pub fn fifo_probation_ratio() -> f64 {
2139            0.1
2140        }
2141
2142        pub fn recover_mode() -> RecoverMode {
2143            RecoverMode::Quiet
2144        }
2145
2146        pub fn runtime_config() -> RuntimeOptions {
2147            RuntimeOptions::Unified(TokioRuntimeOptions::default())
2148        }
2149
2150        pub fn throttle() -> Throttle {
2151            Throttle::new()
2152                .with_iops_counter(foyer::IopsCounter::PerIoSize(
2153                    NonZeroUsize::new(128 * 1024).unwrap(),
2154                ))
2155                .with_read_iops(100000)
2156                .with_write_iops(100000)
2157                .with_write_throughput(1024 * 1024 * 1024)
2158                .with_read_throughput(1024 * 1024 * 1024)
2159        }
2160    }
2161
2162    pub mod cache_refill {
2163        pub fn data_refill_levels() -> Vec<u32> {
2164            vec![]
2165        }
2166
2167        pub fn timeout_ms() -> u64 {
2168            6000
2169        }
2170
2171        pub fn concurrency() -> usize {
2172            10
2173        }
2174
2175        pub fn unit() -> usize {
2176            64
2177        }
2178
2179        pub fn threshold() -> f64 {
2180            0.5
2181        }
2182
2183        pub fn recent_filter_layers() -> usize {
2184            6
2185        }
2186
2187        pub fn recent_filter_rotate_interval_ms() -> usize {
2188            10000
2189        }
2190    }
2191
2192    pub mod heap_profiling {
2193        pub fn enable_auto() -> bool {
2194            true
2195        }
2196
2197        pub fn threshold_auto() -> f32 {
2198            0.9
2199        }
2200
2201        pub fn dir() -> String {
2202            "./".to_owned()
2203        }
2204    }
2205
2206    pub mod developer {
2207        pub fn meta_cached_traces_num() -> u32 {
2208            256
2209        }
2210
2211        pub fn meta_cached_traces_memory_limit_bytes() -> usize {
2212            1 << 27 // 128 MiB
2213        }
2214
2215        pub fn batch_output_channel_size() -> usize {
2216            64
2217        }
2218
2219        pub fn batch_receiver_channel_size() -> usize {
2220            1000
2221        }
2222
2223        pub fn batch_root_stage_channel_size() -> usize {
2224            100
2225        }
2226
2227        pub fn batch_chunk_size() -> usize {
2228            1024
2229        }
2230
2231        pub fn batch_local_execute_buffer_size() -> usize {
2232            64
2233        }
2234
2235        /// Default to unset to be compatible with the behavior before this config is introduced,
2236        /// that is, follow the value of `server.connection_pool_size`.
2237        pub fn batch_exchange_connection_pool_size() -> Option<u16> {
2238            None
2239        }
2240
2241        pub fn stream_enable_executor_row_count() -> bool {
2242            false
2243        }
2244
2245        pub fn connector_message_buffer_size() -> usize {
2246            16
2247        }
2248
2249        pub fn unsafe_stream_extreme_cache_size() -> usize {
2250            10
2251        }
2252
2253        pub fn stream_chunk_size() -> usize {
2254            256
2255        }
2256
2257        pub fn stream_exchange_initial_permits() -> usize {
2258            2048
2259        }
2260
2261        pub fn stream_exchange_batched_permits() -> usize {
2262            256
2263        }
2264
2265        pub fn stream_exchange_concurrent_barriers() -> usize {
2266            1
2267        }
2268
2269        pub fn stream_exchange_concurrent_dispatchers() -> usize {
2270            0
2271        }
2272
2273        pub fn stream_dml_channel_initial_permits() -> usize {
2274            32768
2275        }
2276
2277        pub fn stream_max_barrier_batch_size() -> u32 {
2278            1024
2279        }
2280
2281        pub fn stream_hash_agg_max_dirty_groups_heap_size() -> usize {
2282            64 << 20 // 64MB
2283        }
2284
2285        pub fn enable_trivial_move() -> bool {
2286            true
2287        }
2288
2289        pub fn enable_check_task_level_overlap() -> bool {
2290            false
2291        }
2292
2293        pub fn max_trivial_move_task_count_per_loop() -> usize {
2294            256
2295        }
2296
2297        pub fn max_get_task_probe_times() -> usize {
2298            5
2299        }
2300
2301        pub fn actor_cnt_per_worker_parallelism_soft_limit() -> usize {
2302            100
2303        }
2304
2305        pub fn actor_cnt_per_worker_parallelism_hard_limit() -> usize {
2306            400
2307        }
2308
2309        pub fn hummock_time_travel_sst_info_fetch_batch_size() -> usize {
2310            10_000
2311        }
2312
2313        pub fn hummock_time_travel_sst_info_insert_batch_size() -> usize {
2314            100
2315        }
2316
2317        pub fn time_travel_vacuum_interval_sec() -> u64 {
2318            30
2319        }
2320        pub fn hummock_time_travel_epoch_version_insert_batch_size() -> usize {
2321            1000
2322        }
2323
2324        pub fn hummock_gc_history_insert_batch_size() -> usize {
2325            1000
2326        }
2327
2328        pub fn hummock_time_travel_filter_out_objects_batch_size() -> usize {
2329            1000
2330        }
2331
2332        pub fn hummock_time_travel_filter_out_objects_v1() -> bool {
2333            false
2334        }
2335
2336        pub fn hummock_time_travel_filter_out_objects_list_version_batch_size() -> usize {
2337            10
2338        }
2339
2340        pub fn hummock_time_travel_filter_out_objects_list_delta_batch_size() -> usize {
2341            1000
2342        }
2343
2344        pub fn memory_controller_threshold_aggressive() -> f64 {
2345            0.9
2346        }
2347
2348        pub fn memory_controller_threshold_graceful() -> f64 {
2349            0.81
2350        }
2351
2352        pub fn memory_controller_threshold_stable() -> f64 {
2353            0.72
2354        }
2355
2356        pub fn memory_controller_eviction_factor_aggressive() -> f64 {
2357            2.0
2358        }
2359
2360        pub fn memory_controller_eviction_factor_graceful() -> f64 {
2361            1.5
2362        }
2363
2364        pub fn memory_controller_eviction_factor_stable() -> f64 {
2365            1.0
2366        }
2367
2368        pub fn memory_controller_update_interval_ms() -> usize {
2369            100
2370        }
2371
2372        pub fn memory_controller_sequence_tls_step() -> u64 {
2373            128
2374        }
2375
2376        pub fn memory_controller_sequence_tls_lag() -> u64 {
2377            32
2378        }
2379
2380        pub fn stream_enable_arrangement_backfill() -> bool {
2381            true
2382        }
2383
2384        pub fn enable_shared_source() -> bool {
2385            true
2386        }
2387
2388        pub fn stream_high_join_amplification_threshold() -> usize {
2389            2048
2390        }
2391
2392        /// Default to 1 to be compatible with the behavior before this config is introduced.
2393        pub fn stream_exchange_connection_pool_size() -> Option<u16> {
2394            Some(1)
2395        }
2396
2397        pub fn enable_actor_tokio_metrics() -> bool {
2398            false
2399        }
2400
2401        pub fn stream_enable_auto_schema_change() -> bool {
2402            true
2403        }
2404
2405        pub fn switch_jdbc_pg_to_native() -> bool {
2406            false
2407        }
2408
2409        pub fn streaming_hash_join_entry_state_max_rows() -> usize {
2410            // NOTE(kwannoel): This is just an arbitrary number.
2411            30000
2412        }
2413
2414        pub fn enable_explain_analyze_stats() -> bool {
2415            true
2416        }
2417
2418        pub fn rpc_client_connect_timeout_secs() -> u64 {
2419            5
2420        }
2421
2422        pub fn iceberg_list_interval_sec() -> u64 {
2423            1
2424        }
2425
2426        pub fn iceberg_fetch_batch_size() -> u64 {
2427            1024
2428        }
2429
2430        pub fn iceberg_sink_positional_delete_cache_size() -> usize {
2431            1024
2432        }
2433    }
2434
2435    pub use crate::system_param::default as system;
2436
2437    pub mod batch {
2438        pub fn enable_barrier_read() -> bool {
2439            false
2440        }
2441
2442        pub fn enable_spill() -> bool {
2443            true
2444        }
2445
2446        pub fn statement_timeout_in_sec() -> u32 {
2447            // 1 hour
2448            60 * 60
2449        }
2450
2451        pub fn mask_worker_temporary_secs() -> usize {
2452            30
2453        }
2454
2455        pub fn redact_sql_option_keywords() -> Vec<String> {
2456            [
2457                "credential",
2458                "key",
2459                "password",
2460                "private",
2461                "secret",
2462                "token",
2463            ]
2464            .into_iter()
2465            .map(str::to_string)
2466            .collect()
2467        }
2468    }
2469
2470    pub mod frontend {
2471        pub fn max_total_query_size_bytes() -> u64 {
2472            1024 * 1024 * 1024
2473        }
2474
2475        pub fn min_single_query_size_bytes() -> u64 {
2476            1024 * 1024
2477        }
2478
2479        pub fn max_single_query_size_bytes() -> u64 {
2480            1024 * 1024 * 1024
2481        }
2482    }
2483
2484    pub mod udf {
2485        pub fn enable_embedded_python_udf() -> bool {
2486            false
2487        }
2488
2489        pub fn enable_embedded_javascript_udf() -> bool {
2490            true
2491        }
2492
2493        pub fn enable_embedded_wasm_udf() -> bool {
2494            true
2495        }
2496    }
2497
2498    pub mod compaction_config {
2499        const MB: u64 = 1024 * 1024;
2500        const GB: u64 = 1024 * 1024 * 1024;
2501        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
2502        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
2503        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
2504
2505        // decrease this configure when the generation of checkpoint barrier is not frequent.
2506        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
2507        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
2508        // 32MB
2509        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
2510        const DEFAULT_LEVEL_MULTIPLIER: u64 = 5;
2511        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
2512        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 300;
2513        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
2514        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
2515        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
2516        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
2517        const DEFAULT_EMERGENCY_PICKER: bool = true;
2518        const DEFAULT_MAX_LEVEL: u32 = 6;
2519        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
2520        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
2521        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 64;
2522        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
2523        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
2524        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 10000; // 10000 * 32M = 320G
2525        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
2526
2527        use crate::catalog::hummock::CompactionFilterFlag;
2528
2529        pub fn max_bytes_for_level_base() -> u64 {
2530            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
2531        }
2532
2533        pub fn max_bytes_for_level_multiplier() -> u64 {
2534            DEFAULT_LEVEL_MULTIPLIER
2535        }
2536
2537        pub fn max_compaction_bytes() -> u64 {
2538            DEFAULT_MAX_COMPACTION_BYTES
2539        }
2540
2541        pub fn sub_level_max_compaction_bytes() -> u64 {
2542            DEFAULT_MIN_COMPACTION_BYTES
2543        }
2544
2545        pub fn level0_tier_compact_file_number() -> u64 {
2546            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
2547        }
2548
2549        pub fn target_file_size_base() -> u64 {
2550            DEFAULT_TARGET_FILE_SIZE_BASE
2551        }
2552
2553        pub fn compaction_filter_mask() -> u32 {
2554            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
2555        }
2556
2557        pub fn max_sub_compaction() -> u32 {
2558            DEFAULT_MAX_SUB_COMPACTION
2559        }
2560
2561        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
2562            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
2563        }
2564
2565        pub fn level0_sub_level_compact_level_count() -> u32 {
2566            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
2567        }
2568
2569        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
2570            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
2571        }
2572
2573        pub fn max_space_reclaim_bytes() -> u64 {
2574            DEFAULT_MAX_SPACE_RECLAIM_BYTES
2575        }
2576
2577        pub fn level0_max_compact_file_number() -> u64 {
2578            DEFAULT_MAX_COMPACTION_FILE_COUNT
2579        }
2580
2581        pub fn tombstone_reclaim_ratio() -> u32 {
2582            DEFAULT_TOMBSTONE_RATIO_PERCENT
2583        }
2584
2585        pub fn enable_emergency_picker() -> bool {
2586            DEFAULT_EMERGENCY_PICKER
2587        }
2588
2589        pub fn max_level() -> u32 {
2590            DEFAULT_MAX_LEVEL
2591        }
2592
2593        pub fn max_l0_compact_level_count() -> u32 {
2594            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
2595        }
2596
2597        pub fn sst_allowed_trivial_move_min_size() -> u64 {
2598            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
2599        }
2600
2601        pub fn disable_auto_group_scheduling() -> bool {
2602            false
2603        }
2604
2605        pub fn max_overlapping_level_size() -> u64 {
2606            256 * MB
2607        }
2608
2609        pub fn sst_allowed_trivial_move_max_count() -> u32 {
2610            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
2611        }
2612
2613        pub fn emergency_level0_sst_file_count() -> u32 {
2614            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
2615        }
2616
2617        pub fn emergency_level0_sub_level_partition() -> u32 {
2618            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
2619        }
2620
2621        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
2622            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
2623        }
2624
2625        pub fn level0_stop_write_threshold_max_size() -> u64 {
2626            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
2627        }
2628
2629        pub fn enable_optimize_l0_interval_selection() -> bool {
2630            false
2631        }
2632    }
2633
2634    pub mod object_store_config {
2635        const DEFAULT_REQ_BACKOFF_INTERVAL_MS: u64 = 1000; // 1s
2636        const DEFAULT_REQ_BACKOFF_MAX_DELAY_MS: u64 = 10 * 1000; // 10s
2637        const DEFAULT_REQ_MAX_RETRY_ATTEMPTS: usize = 3;
2638
2639        pub fn set_atomic_write_dir() -> bool {
2640            false
2641        }
2642
2643        pub fn object_store_req_backoff_interval_ms() -> u64 {
2644            DEFAULT_REQ_BACKOFF_INTERVAL_MS
2645        }
2646
2647        pub fn object_store_req_backoff_max_delay_ms() -> u64 {
2648            DEFAULT_REQ_BACKOFF_MAX_DELAY_MS // 10s
2649        }
2650
2651        pub fn object_store_req_backoff_factor() -> u64 {
2652            2
2653        }
2654
2655        pub fn object_store_upload_attempt_timeout_ms() -> u64 {
2656            8 * 1000 // 8s
2657        }
2658
2659        pub fn object_store_upload_retry_attempts() -> usize {
2660            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2661        }
2662
2663        // init + upload_part + finish
2664        pub fn object_store_streaming_upload_attempt_timeout_ms() -> u64 {
2665            5 * 1000 // 5s
2666        }
2667
2668        pub fn object_store_streaming_upload_retry_attempts() -> usize {
2669            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2670        }
2671
2672        // tips: depend on block_size
2673        pub fn object_store_read_attempt_timeout_ms() -> u64 {
2674            8 * 1000 // 8s
2675        }
2676
2677        pub fn object_store_read_retry_attempts() -> usize {
2678            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2679        }
2680
2681        pub fn object_store_streaming_read_attempt_timeout_ms() -> u64 {
2682            3 * 1000 // 3s
2683        }
2684
2685        pub fn object_store_streaming_read_retry_attempts() -> usize {
2686            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2687        }
2688
2689        pub fn object_store_metadata_attempt_timeout_ms() -> u64 {
2690            60 * 1000 // 1min
2691        }
2692
2693        pub fn object_store_metadata_retry_attempts() -> usize {
2694            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2695        }
2696
2697        pub fn object_store_delete_attempt_timeout_ms() -> u64 {
2698            5 * 1000
2699        }
2700
2701        pub fn object_store_delete_retry_attempts() -> usize {
2702            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2703        }
2704
2705        // tips: depend on batch size
2706        pub fn object_store_delete_objects_attempt_timeout_ms() -> u64 {
2707            5 * 1000
2708        }
2709
2710        pub fn object_store_delete_objects_retry_attempts() -> usize {
2711            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2712        }
2713
2714        pub fn object_store_list_attempt_timeout_ms() -> u64 {
2715            10 * 60 * 1000
2716        }
2717
2718        pub fn object_store_list_retry_attempts() -> usize {
2719            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
2720        }
2721
2722        pub fn opendal_upload_concurrency() -> usize {
2723            256
2724        }
2725
2726        pub fn upload_part_size() -> usize {
2727            // 16m
2728            16 * 1024 * 1024
2729        }
2730
2731        pub mod s3 {
2732            const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
2733
2734            const DEFAULT_KEEPALIVE_MS: u64 = 600 * 1000; // 10min
2735
2736            pub fn keepalive_ms() -> Option<u64> {
2737                Some(DEFAULT_KEEPALIVE_MS) // 10min
2738            }
2739
2740            pub fn recv_buffer_size() -> Option<usize> {
2741                Some(1 << 21) // 2m
2742            }
2743
2744            pub fn send_buffer_size() -> Option<usize> {
2745                None
2746            }
2747
2748            pub fn nodelay() -> Option<bool> {
2749                Some(true)
2750            }
2751
2752            pub fn identity_resolution_timeout_s() -> u64 {
2753                DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S
2754            }
2755
2756            pub mod developer {
2757                pub fn retry_unknown_service_error() -> bool {
2758                    false
2759                }
2760
2761                pub fn retryable_service_error_codes() -> Vec<String> {
2762                    vec!["SlowDown".into(), "TooManyRequests".into()]
2763                }
2764
2765                pub fn use_opendal() -> bool {
2766                    true
2767                }
2768            }
2769        }
2770    }
2771
2772    pub mod meta_store_config {
2773        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
2774        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
2775        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
2776        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
2777        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
2778
2779        pub fn max_connections() -> u32 {
2780            DEFAULT_MAX_CONNECTIONS
2781        }
2782
2783        pub fn min_connections() -> u32 {
2784            DEFAULT_MIN_CONNECTIONS
2785        }
2786
2787        pub fn connection_timeout_sec() -> u64 {
2788            DEFAULT_CONNECTION_TIMEOUT_SEC
2789        }
2790
2791        pub fn idle_timeout_sec() -> u64 {
2792            DEFAULT_IDLE_TIMEOUT_SEC
2793        }
2794
2795        pub fn acquire_timeout_sec() -> u64 {
2796            DEFAULT_ACQUIRE_TIMEOUT_SEC
2797        }
2798    }
2799}
2800
2801#[derive(Debug, Clone)]
2802pub enum EvictionConfig {
2803    Lru(LruConfig),
2804    Lfu(LfuConfig),
2805    S3Fifo(S3FifoConfig),
2806}
2807
2808impl EvictionConfig {
2809    pub fn for_test() -> Self {
2810        Self::Lru(LruConfig {
2811            high_priority_pool_ratio: 0.0,
2812        })
2813    }
2814}
2815
2816impl From<EvictionConfig> for foyer::EvictionConfig {
2817    fn from(value: EvictionConfig) -> Self {
2818        match value {
2819            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
2820            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
2821            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
2822        }
2823    }
2824}
2825
2826pub struct StorageMemoryConfig {
2827    pub block_cache_capacity_mb: usize,
2828    pub block_cache_shard_num: usize,
2829    pub meta_cache_capacity_mb: usize,
2830    pub meta_cache_shard_num: usize,
2831    pub vector_block_cache_capacity_mb: usize,
2832    pub vector_block_cache_shard_num: usize,
2833    pub vector_meta_cache_capacity_mb: usize,
2834    pub vector_meta_cache_shard_num: usize,
2835    pub shared_buffer_capacity_mb: usize,
2836    pub compactor_memory_limit_mb: usize,
2837    pub prefetch_buffer_capacity_mb: usize,
2838    pub block_cache_eviction_config: EvictionConfig,
2839    pub meta_cache_eviction_config: EvictionConfig,
2840    pub vector_block_cache_eviction_config: EvictionConfig,
2841    pub vector_meta_cache_eviction_config: EvictionConfig,
2842    pub block_file_cache_flush_buffer_threshold_mb: usize,
2843    pub meta_file_cache_flush_buffer_threshold_mb: usize,
2844}
2845
2846pub const MAX_META_CACHE_SHARD_BITS: usize = 4;
2847pub const MIN_BUFFER_SIZE_PER_SHARD: usize = 256;
2848pub const MAX_BLOCK_CACHE_SHARD_BITS: usize = 6; // It means that there will be 64 shards lru-cache to avoid lock conflict.
2849
2850pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
2851    let block_cache_capacity_mb = s.storage.cache.block_cache_capacity_mb.unwrap_or(
2852        // adapt to old version
2853        s.storage
2854            .block_cache_capacity_mb
2855            .unwrap_or(default::storage::block_cache_capacity_mb()),
2856    );
2857    let meta_cache_capacity_mb = s.storage.cache.meta_cache_capacity_mb.unwrap_or(
2858        // adapt to old version
2859        s.storage
2860            .block_cache_capacity_mb
2861            .unwrap_or(default::storage::meta_cache_capacity_mb()),
2862    );
2863    let shared_buffer_capacity_mb = s
2864        .storage
2865        .shared_buffer_capacity_mb
2866        .unwrap_or(default::storage::shared_buffer_capacity_mb());
2867    let meta_cache_shard_num = s.storage.cache.meta_cache_shard_num.unwrap_or_else(|| {
2868        let mut shard_bits = MAX_META_CACHE_SHARD_BITS;
2869        while (meta_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0 {
2870            shard_bits -= 1;
2871        }
2872        shard_bits
2873    });
2874    let block_cache_shard_num = s.storage.cache.block_cache_shard_num.unwrap_or_else(|| {
2875        let mut shard_bits = MAX_BLOCK_CACHE_SHARD_BITS;
2876        while (block_cache_capacity_mb >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0
2877        {
2878            shard_bits -= 1;
2879        }
2880        shard_bits
2881    });
2882    let compactor_memory_limit_mb = s
2883        .storage
2884        .compactor_memory_limit_mb
2885        .unwrap_or(default::storage::compactor_memory_limit_mb());
2886
2887    let get_eviction_config = |c: &CacheEvictionConfig| {
2888        match c {
2889            CacheEvictionConfig::Lru {
2890                high_priority_ratio_in_percent,
2891            } => EvictionConfig::Lru(LruConfig {
2892                high_priority_pool_ratio: high_priority_ratio_in_percent.unwrap_or(
2893                    // adapt to old version
2894                    s.storage
2895                        .high_priority_ratio_in_percent
2896                        .unwrap_or(default::storage::high_priority_ratio_in_percent()),
2897                ) as f64
2898                    / 100.0,
2899            }),
2900            CacheEvictionConfig::Lfu {
2901                window_capacity_ratio_in_percent,
2902                protected_capacity_ratio_in_percent,
2903                cmsketch_eps,
2904                cmsketch_confidence,
2905            } => EvictionConfig::Lfu(LfuConfig {
2906                window_capacity_ratio: window_capacity_ratio_in_percent
2907                    .unwrap_or(default::storage::window_capacity_ratio_in_percent())
2908                    as f64
2909                    / 100.0,
2910                protected_capacity_ratio: protected_capacity_ratio_in_percent
2911                    .unwrap_or(default::storage::protected_capacity_ratio_in_percent())
2912                    as f64
2913                    / 100.0,
2914                cmsketch_eps: cmsketch_eps.unwrap_or(default::storage::cmsketch_eps()),
2915                cmsketch_confidence: cmsketch_confidence
2916                    .unwrap_or(default::storage::cmsketch_confidence()),
2917            }),
2918            CacheEvictionConfig::S3Fifo {
2919                small_queue_capacity_ratio_in_percent,
2920                ghost_queue_capacity_ratio_in_percent,
2921                small_to_main_freq_threshold,
2922            } => EvictionConfig::S3Fifo(S3FifoConfig {
2923                small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
2924                    .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
2925                    as f64
2926                    / 100.0,
2927                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
2928                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
2929                    as f64
2930                    / 100.0,
2931                small_to_main_freq_threshold: small_to_main_freq_threshold
2932                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
2933            }),
2934        }
2935    };
2936
2937    let block_cache_eviction_config = get_eviction_config(&s.storage.cache.block_cache_eviction);
2938    let meta_cache_eviction_config = get_eviction_config(&s.storage.cache.meta_cache_eviction);
2939    let vector_block_cache_eviction_config =
2940        get_eviction_config(&s.storage.cache.vector_block_cache_eviction_config);
2941    let vector_meta_cache_eviction_config =
2942        get_eviction_config(&s.storage.cache.vector_meta_cache_eviction_config);
2943
2944    let prefetch_buffer_capacity_mb =
2945        s.storage
2946            .shared_buffer_capacity_mb
2947            .unwrap_or(match &block_cache_eviction_config {
2948                EvictionConfig::Lru(lru) => {
2949                    ((1.0 - lru.high_priority_pool_ratio) * block_cache_capacity_mb as f64) as usize
2950                }
2951                EvictionConfig::Lfu(lfu) => {
2952                    ((1.0 - lfu.protected_capacity_ratio) * block_cache_capacity_mb as f64) as usize
2953                }
2954                EvictionConfig::S3Fifo(s3fifo) => {
2955                    (s3fifo.small_queue_capacity_ratio * block_cache_capacity_mb as f64) as usize
2956                }
2957            });
2958
2959    let block_file_cache_flush_buffer_threshold_mb = s
2960        .storage
2961        .data_file_cache
2962        .flush_buffer_threshold_mb
2963        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
2964    let meta_file_cache_flush_buffer_threshold_mb = s
2965        .storage
2966        .meta_file_cache
2967        .flush_buffer_threshold_mb
2968        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
2969
2970    StorageMemoryConfig {
2971        block_cache_capacity_mb,
2972        block_cache_shard_num,
2973        meta_cache_capacity_mb,
2974        meta_cache_shard_num,
2975        vector_block_cache_capacity_mb: s.storage.cache.vector_block_cache_capacity_mb,
2976        vector_block_cache_shard_num: s.storage.cache.vector_block_cache_shard_num,
2977        vector_meta_cache_capacity_mb: s.storage.cache.vector_meta_cache_capacity_mb,
2978        vector_meta_cache_shard_num: s.storage.cache.vector_meta_cache_shard_num,
2979        shared_buffer_capacity_mb,
2980        compactor_memory_limit_mb,
2981        prefetch_buffer_capacity_mb,
2982        block_cache_eviction_config,
2983        meta_cache_eviction_config,
2984        vector_block_cache_eviction_config,
2985        vector_meta_cache_eviction_config,
2986        block_file_cache_flush_buffer_threshold_mb,
2987        meta_file_cache_flush_buffer_threshold_mb,
2988    }
2989}
2990
2991#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
2992pub struct CompactionConfig {
2993    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
2994    pub max_bytes_for_level_base: u64,
2995    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
2996    pub max_bytes_for_level_multiplier: u64,
2997    #[serde(default = "default::compaction_config::max_compaction_bytes")]
2998    pub max_compaction_bytes: u64,
2999    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
3000    pub sub_level_max_compaction_bytes: u64,
3001    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
3002    pub level0_tier_compact_file_number: u64,
3003    #[serde(default = "default::compaction_config::target_file_size_base")]
3004    pub target_file_size_base: u64,
3005    #[serde(default = "default::compaction_config::compaction_filter_mask")]
3006    pub compaction_filter_mask: u32,
3007    #[serde(default = "default::compaction_config::max_sub_compaction")]
3008    pub max_sub_compaction: u32,
3009    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
3010    pub level0_stop_write_threshold_sub_level_number: u64,
3011    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
3012    pub level0_sub_level_compact_level_count: u32,
3013    #[serde(
3014        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
3015    )]
3016    pub level0_overlapping_sub_level_compact_level_count: u32,
3017    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
3018    pub max_space_reclaim_bytes: u64,
3019    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
3020    pub level0_max_compact_file_number: u64,
3021    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
3022    pub tombstone_reclaim_ratio: u32,
3023    #[serde(default = "default::compaction_config::enable_emergency_picker")]
3024    pub enable_emergency_picker: bool,
3025    #[serde(default = "default::compaction_config::max_level")]
3026    pub max_level: u32,
3027    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
3028    pub sst_allowed_trivial_move_min_size: u64,
3029    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
3030    pub sst_allowed_trivial_move_max_count: u32,
3031    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
3032    pub max_l0_compact_level_count: u32,
3033    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
3034    pub disable_auto_group_scheduling: bool,
3035    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
3036    pub max_overlapping_level_size: u64,
3037    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
3038    pub emergency_level0_sst_file_count: u32,
3039    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
3040    pub emergency_level0_sub_level_partition: u32,
3041    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
3042    pub level0_stop_write_threshold_max_sst_count: u32,
3043    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
3044    pub level0_stop_write_threshold_max_size: u64,
3045    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
3046    pub enable_optimize_l0_interval_selection: bool,
3047}
3048
3049/// Note: only applies to meta store backends other than `SQLite`.
3050#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
3051pub struct MetaStoreConfig {
3052    /// Maximum number of connections for the meta store connection pool.
3053    #[serde(default = "default::meta_store_config::max_connections")]
3054    pub max_connections: u32,
3055    /// Minimum number of connections for the meta store connection pool.
3056    #[serde(default = "default::meta_store_config::min_connections")]
3057    pub min_connections: u32,
3058    /// Connection timeout in seconds for a meta store connection.
3059    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
3060    pub connection_timeout_sec: u64,
3061    /// Idle timeout in seconds for a meta store connection.
3062    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
3063    pub idle_timeout_sec: u64,
3064    /// Acquire timeout in seconds for a meta store connection.
3065    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
3066    pub acquire_timeout_sec: u64,
3067}
3068
3069#[cfg(test)]
3070mod tests {
3071    use risingwave_license::LicenseKey;
3072
3073    use super::*;
3074
3075    fn default_config_for_docs() -> RwConfig {
3076        let mut config = RwConfig::default();
3077        // Set `license_key` to empty in the docs to avoid any confusion.
3078        config.system.license_key = Some(LicenseKey::empty());
3079        config
3080    }
3081
3082    /// This test ensures that `config/example.toml` is up-to-date with the default values specified
3083    /// in this file. Developer should run `./risedev generate-example-config` to update it if this
3084    /// test fails.
3085    #[test]
3086    fn test_example_up_to_date() {
3087        const HEADER: &str = "# This file is generated by ./risedev generate-example-config
3088# Check detailed comments in src/common/src/config.rs";
3089
3090        let actual = expect_test::expect_file!["../../config/example.toml"];
3091        let default = toml::to_string(&default_config_for_docs()).expect("failed to serialize");
3092
3093        let expected = format!("{HEADER}\n\n{default}");
3094        actual.assert_eq(&expected);
3095
3096        let expected = rw_config_to_markdown();
3097        let actual = expect_test::expect_file!["../../config/docs.md"];
3098        actual.assert_eq(&expected);
3099    }
3100
3101    #[derive(Debug)]
3102    struct ConfigItemDoc {
3103        desc: String,
3104        default: String,
3105    }
3106
3107    fn rw_config_to_markdown() -> String {
3108        let mut config_rustdocs = BTreeMap::<String, Vec<(String, String)>>::new();
3109        RwConfig::config_docs("".to_owned(), &mut config_rustdocs);
3110
3111        // Section -> Config Name -> ConfigItemDoc
3112        let mut configs: BTreeMap<String, BTreeMap<String, ConfigItemDoc>> = config_rustdocs
3113            .into_iter()
3114            .map(|(k, v)| {
3115                let docs: BTreeMap<String, ConfigItemDoc> = v
3116                    .into_iter()
3117                    .map(|(name, desc)| {
3118                        (
3119                            name,
3120                            ConfigItemDoc {
3121                                desc,
3122                                default: "".to_owned(), // unset
3123                            },
3124                        )
3125                    })
3126                    .collect();
3127                (k, docs)
3128            })
3129            .collect();
3130
3131        let toml_doc: BTreeMap<String, toml::Value> =
3132            toml::from_str(&toml::to_string(&default_config_for_docs()).unwrap()).unwrap();
3133        toml_doc.into_iter().for_each(|(name, value)| {
3134            set_default_values("".to_owned(), name, value, &mut configs);
3135        });
3136
3137        let mut markdown = "# RisingWave System Configurations\n\n".to_owned()
3138            + "This page is automatically generated by `./risedev generate-example-config`\n";
3139        for (section, configs) in configs {
3140            if configs.is_empty() {
3141                continue;
3142            }
3143            markdown.push_str(&format!("\n## {}\n\n", section));
3144            markdown.push_str("| Config | Description | Default |\n");
3145            markdown.push_str("|--------|-------------|---------|\n");
3146            for (config, doc) in configs {
3147                markdown.push_str(&format!(
3148                    "| {} | {} | {} |\n",
3149                    config, doc.desc, doc.default
3150                ));
3151            }
3152        }
3153        markdown
3154    }
3155
3156    fn set_default_values(
3157        section: String,
3158        name: String,
3159        value: toml::Value,
3160        configs: &mut BTreeMap<String, BTreeMap<String, ConfigItemDoc>>,
3161    ) {
3162        // Set the default value if it's a config name-value pair, otherwise it's a sub-section (Table) that should be recursively processed.
3163        if let toml::Value::Table(table) = value {
3164            let section_configs: BTreeMap<String, toml::Value> =
3165                table.clone().into_iter().collect();
3166            let sub_section = if section.is_empty() {
3167                name
3168            } else {
3169                format!("{}.{}", section, name)
3170            };
3171            section_configs
3172                .into_iter()
3173                .for_each(|(k, v)| set_default_values(sub_section.clone(), k, v, configs))
3174        } else if let Some(t) = configs.get_mut(&section)
3175            && let Some(item_doc) = t.get_mut(&name)
3176        {
3177            item_doc.default = format!("{}", value);
3178        }
3179    }
3180
3181    #[test]
3182    fn test_object_store_configs_backward_compatibility() {
3183        // Define configs with the old name and make sure it still works
3184        {
3185            let config: RwConfig = toml::from_str(
3186                r#"
3187            [storage.object_store]
3188            object_store_set_atomic_write_dir = true
3189
3190            [storage.object_store.s3]
3191            object_store_keepalive_ms = 1
3192            object_store_send_buffer_size = 1
3193            object_store_recv_buffer_size = 1
3194            object_store_nodelay = false
3195
3196            [storage.object_store.s3.developer]
3197            object_store_retry_unknown_service_error = true
3198            object_store_retryable_service_error_codes = ['dummy']
3199
3200
3201            "#,
3202            )
3203            .unwrap();
3204
3205            assert!(config.storage.object_store.set_atomic_write_dir);
3206            assert_eq!(config.storage.object_store.s3.keepalive_ms, Some(1));
3207            assert_eq!(config.storage.object_store.s3.send_buffer_size, Some(1));
3208            assert_eq!(config.storage.object_store.s3.recv_buffer_size, Some(1));
3209            assert_eq!(config.storage.object_store.s3.nodelay, Some(false));
3210            assert!(
3211                config
3212                    .storage
3213                    .object_store
3214                    .s3
3215                    .developer
3216                    .retry_unknown_service_error
3217            );
3218            assert_eq!(
3219                config
3220                    .storage
3221                    .object_store
3222                    .s3
3223                    .developer
3224                    .retryable_service_error_codes,
3225                vec!["dummy".to_owned()]
3226            );
3227        }
3228
3229        // Define configs with the new name and make sure it works
3230        {
3231            let config: RwConfig = toml::from_str(
3232                r#"
3233            [storage.object_store]
3234            set_atomic_write_dir = true
3235
3236            [storage.object_store.s3]
3237            keepalive_ms = 1
3238            send_buffer_size = 1
3239            recv_buffer_size = 1
3240            nodelay = false
3241
3242            [storage.object_store.s3.developer]
3243            retry_unknown_service_error = true
3244            retryable_service_error_codes = ['dummy']
3245
3246
3247            "#,
3248            )
3249            .unwrap();
3250
3251            assert!(config.storage.object_store.set_atomic_write_dir);
3252            assert_eq!(config.storage.object_store.s3.keepalive_ms, Some(1));
3253            assert_eq!(config.storage.object_store.s3.send_buffer_size, Some(1));
3254            assert_eq!(config.storage.object_store.s3.recv_buffer_size, Some(1));
3255            assert_eq!(config.storage.object_store.s3.nodelay, Some(false));
3256            assert!(
3257                config
3258                    .storage
3259                    .object_store
3260                    .s3
3261                    .developer
3262                    .retry_unknown_service_error
3263            );
3264            assert_eq!(
3265                config
3266                    .storage
3267                    .object_store
3268                    .s3
3269                    .developer
3270                    .retryable_service_error_codes,
3271                vec!["dummy".to_owned()]
3272            );
3273        }
3274    }
3275
3276    #[test]
3277    fn test_meta_configs_backward_compatibility() {
3278        // Test periodic_space_reclaim_compaction_interval_sec
3279        {
3280            let config: RwConfig = toml::from_str(
3281                r#"
3282            [meta]
3283            periodic_split_compact_group_interval_sec = 1
3284            table_write_throughput_threshold = 10
3285            min_table_split_write_throughput = 5
3286            "#,
3287            )
3288            .unwrap();
3289
3290            assert_eq!(
3291                config
3292                    .meta
3293                    .periodic_scheduling_compaction_group_split_interval_sec,
3294                1
3295            );
3296            assert_eq!(config.meta.table_high_write_throughput_threshold, 10);
3297            assert_eq!(config.meta.table_low_write_throughput_threshold, 5);
3298        }
3299    }
3300}
risingwave_common/config.rs

risingwave_common/
config.rs
