risingwave_common/config/
meta.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16
17#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
18pub enum MetaBackend {
19    #[default]
20    Mem,
21    Sql, // any database url
22    Sqlite,
23    Postgres,
24    Mysql,
25}
26
27#[derive(Copy, Clone, Debug, Default)]
28pub enum DefaultParallelism {
29    #[default]
30    Full,
31    Default(NonZeroUsize),
32}
33
34impl Serialize for DefaultParallelism {
35    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
36    where
37        S: Serializer,
38    {
39        #[derive(Debug, Serialize, Deserialize)]
40        #[serde(untagged)]
41        enum Parallelism {
42            Str(String),
43            Int(usize),
44        }
45        match self {
46            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
47            DefaultParallelism::Default(val) => {
48                Parallelism::Int(val.get() as _).serialize(serializer)
49            }
50        }
51    }
52}
53
54impl<'de> Deserialize<'de> for DefaultParallelism {
55    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
56    where
57        D: serde::Deserializer<'de>,
58    {
59        #[derive(Debug, Deserialize)]
60        #[serde(untagged)]
61        enum Parallelism {
62            Str(String),
63            Int(usize),
64        }
65        let p = Parallelism::deserialize(deserializer)?;
66        match p {
67            Parallelism::Str(s) => {
68                if s.trim().eq_ignore_ascii_case("full") {
69                    Ok(DefaultParallelism::Full)
70                } else {
71                    Err(serde::de::Error::custom(format!(
72                        "invalid default parallelism: {}",
73                        s
74                    )))
75                }
76            }
77            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
78                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
79                // here because it requires extra context. The check will be done when scheduling jobs.
80                NonZeroUsize::new(i).ok_or_else(|| {
81                    serde::de::Error::custom("default parallelism should not be 0")
82                })?,
83            )),
84        }
85    }
86}
87
88/// The section `[meta]` in `risingwave.toml`.
89#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
90pub struct MetaConfig {
91    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
92    /// are dangling.
93    #[serde(default = "default::meta::min_sst_retention_time_sec")]
94    pub min_sst_retention_time_sec: u64,
95
96    /// Interval of automatic hummock full GC.
97    #[serde(default = "default::meta::full_gc_interval_sec")]
98    pub full_gc_interval_sec: u64,
99
100    /// Max number of object per full GC job can fetch.
101    #[serde(default = "default::meta::full_gc_object_limit")]
102    pub full_gc_object_limit: u64,
103
104    /// Duration in seconds to retain garbage collection history data.
105    #[serde(default = "default::meta::gc_history_retention_time_sec")]
106    pub gc_history_retention_time_sec: u64,
107
108    /// Max number of inflight time travel query.
109    #[serde(default = "default::meta::max_inflight_time_travel_query")]
110    pub max_inflight_time_travel_query: u64,
111
112    /// Schedule compaction for all compaction groups with this interval.
113    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
114    pub periodic_compaction_interval_sec: u64,
115
116    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
117    /// from object store.
118    #[serde(default = "default::meta::vacuum_interval_sec")]
119    pub vacuum_interval_sec: u64,
120
121    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
122    /// meta node.
123    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
124    pub vacuum_spin_interval_ms: u64,
125
126    /// Interval of invoking iceberg garbage collection, to expire old snapshots.
127    #[serde(default = "default::meta::iceberg_gc_interval_sec")]
128    pub iceberg_gc_interval_sec: u64,
129
130    /// Interval of hummock version checkpoint.
131    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
132    pub hummock_version_checkpoint_interval_sec: u64,
133
134    /// If enabled, `SSTable` object file and version delta will be retained.
135    ///
136    /// `SSTable` object file need to be deleted via full GC.
137    ///
138    /// version delta need to be manually deleted.
139    #[serde(default = "default::meta::enable_hummock_data_archive")]
140    pub enable_hummock_data_archive: bool,
141
142    /// The interval at which a Hummock version snapshot is taken for time travel.
143    ///
144    /// Larger value indicates less storage overhead but worse query performance.
145    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
146    pub hummock_time_travel_snapshot_interval: u64,
147
148    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
149    /// attempt is rejected.
150    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
151    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
152
153    /// Maximum allowed heartbeat interval in seconds.
154    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
155    pub max_heartbeat_interval_secs: u32,
156
157    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
158    #[serde(default)]
159    pub disable_recovery: bool,
160
161    /// Whether to disable adaptive-scaling feature.
162    #[serde(default)]
163    pub disable_automatic_parallelism_control: bool,
164
165    /// The number of streaming jobs per scaling operation.
166    #[serde(default = "default::meta::parallelism_control_batch_size")]
167    pub parallelism_control_batch_size: usize,
168
169    /// The period of parallelism control trigger.
170    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
171    pub parallelism_control_trigger_period_sec: u64,
172
173    /// The first delay of parallelism control.
174    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
175    pub parallelism_control_trigger_first_delay_sec: u64,
176
177    #[serde(default = "default::meta::meta_leader_lease_secs")]
178    pub meta_leader_lease_secs: u64,
179
180    /// After specified seconds of idle (no mview or flush), the process will be exited.
181    /// It is mainly useful for playgrounds.
182    #[serde(default)]
183    pub dangerous_max_idle_secs: Option<u64>,
184
185    /// The default global parallelism for all streaming jobs, if user doesn't specify the
186    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
187    /// otherwise it's a number.
188    #[serde(default = "default::meta::default_parallelism")]
189    pub default_parallelism: DefaultParallelism,
190
191    /// Whether to enable deterministic compaction scheduling, which
192    /// will disable all auto scheduling of compaction tasks.
193    /// Should only be used in e2e tests.
194    #[serde(default)]
195    pub enable_compaction_deterministic: bool,
196
197    /// Enable sanity check when SSTs are committed.
198    #[serde(default)]
199    pub enable_committed_sst_sanity_check: bool,
200
201    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
202    pub node_num_monitor_interval_sec: u64,
203
204    #[serde(default = "default::meta::backend")]
205    pub backend: MetaBackend,
206
207    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
208    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
209    pub periodic_space_reclaim_compaction_interval_sec: u64,
210
211    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
212    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
213    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
214
215    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
216    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
217
218    #[serde(default = "default::meta::move_table_size_limit")]
219    #[deprecated]
220    pub move_table_size_limit: u64,
221
222    #[serde(default = "default::meta::split_group_size_limit")]
223    #[deprecated]
224    pub split_group_size_limit: u64,
225
226    #[serde(default = "default::meta::cut_table_size_limit")]
227    #[deprecated]
228    pub cut_table_size_limit: u64,
229
230    /// Whether to protect dropping a table with incoming sink.
231    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
232    pub protect_drop_table_with_incoming_sink: bool,
233
234    #[serde(default, flatten)]
235    #[config_doc(omitted)]
236    pub unrecognized: Unrecognized<Self>,
237
238    /// Whether config object storage bucket lifecycle to purge stale data.
239    #[serde(default)]
240    pub do_not_config_object_storage_lifecycle: bool,
241
242    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
243    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
244    #[serde(default = "default::meta::partition_vnode_count")]
245    pub partition_vnode_count: u32,
246
247    /// The threshold of write throughput to trigger a group split.
248    #[serde(
249        default = "default::meta::table_high_write_throughput_threshold",
250        alias = "table_write_throughput_threshold"
251    )]
252    pub table_high_write_throughput_threshold: u64,
253
254    #[serde(
255        default = "default::meta::table_low_write_throughput_threshold",
256        alias = "min_table_split_write_throughput"
257    )]
258    /// The threshold of write throughput to trigger a group merge.
259    pub table_low_write_throughput_threshold: u64,
260
261    // If the compaction task does not report heartbeat beyond the
262    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
263    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
264    pub compaction_task_max_heartbeat_interval_secs: u64,
265
266    // If the compaction task does not change in progress beyond the
267    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
268    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
269    pub compaction_task_max_progress_interval_secs: u64,
270
271    #[serde(default)]
272    #[config_doc(nested)]
273    pub compaction_config: CompactionConfig,
274
275    /// Count of partitions of tables in default group and materialized view group.
276    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
277    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
278    /// Set it zero to disable this feature.
279    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
280    pub hybrid_partition_vnode_count: u32,
281
282    #[serde(default = "default::meta::event_log_enabled")]
283    pub event_log_enabled: bool,
284    /// Keeps the latest N events per channel.
285    #[serde(default = "default::meta::event_log_channel_max_size")]
286    pub event_log_channel_max_size: u32,
287
288    #[serde(default, with = "meta_prefix")]
289    #[config_doc(omitted)]
290    pub developer: MetaDeveloperConfig,
291    /// Whether compactor should rewrite row to remove dropped column.
292    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
293    pub enable_dropped_column_reclaim: bool,
294
295    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
296    #[serde(default = "default::meta::split_group_size_ratio")]
297    pub split_group_size_ratio: f64,
298
299    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
300    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
301    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
302    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
303    pub table_stat_high_write_throughput_ratio_for_split: f64,
304
305    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
306    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
307    pub table_stat_low_write_throughput_ratio_for_merge: f64,
308
309    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
310    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
311    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
312    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
313    /// The window seconds of table throughput statistic history for split compaction group.
314    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
315    pub table_stat_throuput_window_seconds_for_split: usize,
316
317    /// The window seconds of table throughput statistic history for merge compaction group.
318    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
319    pub table_stat_throuput_window_seconds_for_merge: usize,
320
321    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
322    /// Set it max value of 64-bit number to disable this feature.
323    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
324    pub compact_task_table_size_partition_threshold_low: u64,
325
326    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
327    /// Set it max value of 64-bit number to disable this feature.
328    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
329    pub compact_task_table_size_partition_threshold_high: u64,
330
331    /// The interval of the periodic scheduling compaction group split job.
332    #[serde(
333        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
334        alias = "periodic_split_compact_group_interval_sec"
335    )]
336    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
337
338    /// The interval of the periodic scheduling compaction group merge job.
339    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
340    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
341
342    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
343    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
344    pub compaction_group_merge_dimension_threshold: f64,
345
346    /// The interval that the CDC table splits initialization should yield to avoid overloading upstream system.
347    #[serde(default = "default::meta::cdc_table_split_init_sleep_interval_splits")]
348    pub cdc_table_split_init_sleep_interval_splits: u64,
349
350    /// The duration that the CDC table splits initialization should yield to avoid overloading upstream system.
351    #[serde(default = "default::meta::cdc_table_split_init_sleep_duration_millis")]
352    pub cdc_table_split_init_sleep_duration_millis: u64,
353
354    /// The batch size that the CDC table splits initialization should use when persisting to meta store.
355    #[serde(default = "default::meta::cdc_table_split_init_insert_batch_size")]
356    pub cdc_table_split_init_insert_batch_size: u64,
357
358    /// Whether to automatically migrate legacy table fragments when meta starts.
359    #[serde(default = "default::meta::enable_legacy_table_migration")]
360    pub enable_legacy_table_migration: bool,
361
362    #[serde(default)]
363    #[config_doc(nested)]
364    pub meta_store_config: MetaStoreConfig,
365}
366
367serde_with::with_prefix!(meta_prefix "meta_");
368
369/// Note: only applies to meta store backends other than `SQLite`.
370#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
371pub struct MetaStoreConfig {
372    /// Maximum number of connections for the meta store connection pool.
373    #[serde(default = "default::meta_store_config::max_connections")]
374    pub max_connections: u32,
375    /// Minimum number of connections for the meta store connection pool.
376    #[serde(default = "default::meta_store_config::min_connections")]
377    pub min_connections: u32,
378    /// Connection timeout in seconds for a meta store connection.
379    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
380    pub connection_timeout_sec: u64,
381    /// Idle timeout in seconds for a meta store connection.
382    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
383    pub idle_timeout_sec: u64,
384    /// Acquire timeout in seconds for a meta store connection.
385    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
386    pub acquire_timeout_sec: u64,
387}
388
389/// The subsections `[meta.developer]`.
390///
391/// It is put at [`MetaConfig::developer`].
392#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
393pub struct MetaDeveloperConfig {
394    /// The number of traces to be cached in-memory by the tracing collector
395    /// embedded in the meta node.
396    #[serde(default = "default::developer::meta_cached_traces_num")]
397    pub cached_traces_num: u32,
398
399    /// The maximum memory usage in bytes for the tracing collector embedded
400    /// in the meta node.
401    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
402    pub cached_traces_memory_limit_bytes: usize,
403
404    /// Compaction picker config
405    #[serde(default = "default::developer::enable_trivial_move")]
406    pub enable_trivial_move: bool,
407    #[serde(default = "default::developer::enable_check_task_level_overlap")]
408    pub enable_check_task_level_overlap: bool,
409    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
410    pub max_trivial_move_task_count_per_loop: usize,
411
412    #[serde(default = "default::developer::max_get_task_probe_times")]
413    pub max_get_task_probe_times: usize,
414
415    /// Max number of actor allowed per parallelism (default = 100).
416    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
417    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
418    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
419
420    /// Max number of actor allowed per parallelism (default = 400).
421    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
422    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
423    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
424
425    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
426    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
427    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
428
429    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
430    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
431    pub hummock_time_travel_sst_info_insert_batch_size: usize,
432
433    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
434    pub time_travel_vacuum_interval_sec: u64,
435
436    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
437    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
438    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
439
440    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
441    pub hummock_gc_history_insert_batch_size: usize,
442
443    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
444    pub hummock_time_travel_filter_out_objects_batch_size: usize,
445
446    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
447    pub hummock_time_travel_filter_out_objects_v1: bool,
448
449    #[serde(
450        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
451    )]
452    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
453
454    #[serde(
455        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
456    )]
457    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
458
459    #[serde(default)]
460    pub compute_client_config: RpcClientConfig,
461
462    #[serde(default)]
463    pub stream_client_config: RpcClientConfig,
464
465    #[serde(default)]
466    pub frontend_client_config: RpcClientConfig,
467}
468
469#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
470pub struct CompactionConfig {
471    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
472    pub max_bytes_for_level_base: u64,
473    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
474    pub max_bytes_for_level_multiplier: u64,
475    #[serde(default = "default::compaction_config::max_compaction_bytes")]
476    pub max_compaction_bytes: u64,
477    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
478    pub sub_level_max_compaction_bytes: u64,
479    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
480    pub level0_tier_compact_file_number: u64,
481    #[serde(default = "default::compaction_config::target_file_size_base")]
482    pub target_file_size_base: u64,
483    #[serde(default = "default::compaction_config::compaction_filter_mask")]
484    pub compaction_filter_mask: u32,
485    #[serde(default = "default::compaction_config::max_sub_compaction")]
486    pub max_sub_compaction: u32,
487    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
488    pub level0_stop_write_threshold_sub_level_number: u64,
489    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
490    pub level0_sub_level_compact_level_count: u32,
491    #[serde(
492        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
493    )]
494    pub level0_overlapping_sub_level_compact_level_count: u32,
495    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
496    pub max_space_reclaim_bytes: u64,
497    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
498    pub level0_max_compact_file_number: u64,
499    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
500    pub tombstone_reclaim_ratio: u32,
501    #[serde(default = "default::compaction_config::enable_emergency_picker")]
502    pub enable_emergency_picker: bool,
503    #[serde(default = "default::compaction_config::max_level")]
504    pub max_level: u32,
505    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
506    pub sst_allowed_trivial_move_min_size: u64,
507    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
508    pub sst_allowed_trivial_move_max_count: u32,
509    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
510    pub max_l0_compact_level_count: u32,
511    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
512    pub disable_auto_group_scheduling: bool,
513    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
514    pub max_overlapping_level_size: u64,
515    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
516    pub emergency_level0_sst_file_count: u32,
517    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
518    pub emergency_level0_sub_level_partition: u32,
519    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
520    pub level0_stop_write_threshold_max_sst_count: u32,
521    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
522    pub level0_stop_write_threshold_max_size: u64,
523    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
524    pub enable_optimize_l0_interval_selection: bool,
525}
526
527pub mod default {
528    pub use crate::config::default::developer;
529
530    pub mod meta {
531        use crate::config::{DefaultParallelism, MetaBackend};
532
533        pub fn min_sst_retention_time_sec() -> u64 {
534            3600 * 6
535        }
536
537        pub fn gc_history_retention_time_sec() -> u64 {
538            3600 * 6
539        }
540
541        pub fn full_gc_interval_sec() -> u64 {
542            3600
543        }
544
545        pub fn full_gc_object_limit() -> u64 {
546            100_000
547        }
548
549        pub fn max_inflight_time_travel_query() -> u64 {
550            1000
551        }
552
553        pub fn periodic_compaction_interval_sec() -> u64 {
554            60
555        }
556
557        pub fn vacuum_interval_sec() -> u64 {
558            30
559        }
560
561        pub fn vacuum_spin_interval_ms() -> u64 {
562            100
563        }
564
565        pub fn iceberg_gc_interval_sec() -> u64 {
566            3600
567        }
568
569        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
570            30
571        }
572
573        pub fn enable_hummock_data_archive() -> bool {
574            false
575        }
576
577        pub fn hummock_time_travel_snapshot_interval() -> u64 {
578            100
579        }
580
581        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
582            10
583        }
584
585        pub fn max_heartbeat_interval_sec() -> u32 {
586            60
587        }
588
589        pub fn meta_leader_lease_secs() -> u64 {
590            30
591        }
592
593        pub fn default_parallelism() -> DefaultParallelism {
594            DefaultParallelism::Full
595        }
596
597        pub fn node_num_monitor_interval_sec() -> u64 {
598            10
599        }
600
601        pub fn backend() -> MetaBackend {
602            MetaBackend::Mem
603        }
604
605        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
606            3600 // 60min
607        }
608
609        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
610            1800 // 30mi
611        }
612
613        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
614            10 // 10s
615        }
616
617        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
618            600
619        }
620
621        // limit the size of state table to trigger split by high throughput
622        pub fn move_table_size_limit() -> u64 {
623            10 * 1024 * 1024 * 1024 // 10GB
624        }
625
626        // limit the size of group to trigger split by group_size and avoid too many small groups
627        pub fn split_group_size_limit() -> u64 {
628            64 * 1024 * 1024 * 1024 // 64GB
629        }
630
631        pub fn protect_drop_table_with_incoming_sink() -> bool {
632            false
633        }
634
635        pub fn partition_vnode_count() -> u32 {
636            16
637        }
638
639        pub fn table_high_write_throughput_threshold() -> u64 {
640            16 * 1024 * 1024 // 16MB
641        }
642
643        pub fn table_low_write_throughput_threshold() -> u64 {
644            4 * 1024 * 1024 // 4MB
645        }
646
647        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
648            30 // 30s
649        }
650
651        pub fn compaction_task_max_progress_interval_secs() -> u64 {
652            60 * 10 // 10min
653        }
654
655        pub fn cut_table_size_limit() -> u64 {
656            1024 * 1024 * 1024 // 1GB
657        }
658
659        pub fn hybrid_partition_vnode_count() -> u32 {
660            4
661        }
662
663        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
664            128 * 1024 * 1024 // 128MB
665        }
666
667        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
668            512 * 1024 * 1024 // 512MB
669        }
670
671        pub fn event_log_enabled() -> bool {
672            true
673        }
674
675        pub fn event_log_channel_max_size() -> u32 {
676            10
677        }
678
679        pub fn parallelism_control_batch_size() -> usize {
680            10
681        }
682
683        pub fn parallelism_control_trigger_period_sec() -> u64 {
684            10
685        }
686
687        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
688            30
689        }
690
691        pub fn enable_dropped_column_reclaim() -> bool {
692            false
693        }
694
695        pub fn split_group_size_ratio() -> f64 {
696            0.9
697        }
698
699        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
700            0.5
701        }
702
703        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
704            0.7
705        }
706
707        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
708            60
709        }
710
711        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
712            240
713        }
714
715        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
716            60 * 10 // 10min
717        }
718
719        pub fn compaction_group_merge_dimension_threshold() -> f64 {
720            1.2
721        }
722
723        pub fn cdc_table_split_init_sleep_interval_splits() -> u64 {
724            1000
725        }
726
727        pub fn cdc_table_split_init_sleep_duration_millis() -> u64 {
728            500
729        }
730
731        pub fn cdc_table_split_init_insert_batch_size() -> u64 {
732            100
733        }
734
735        pub fn enable_legacy_table_migration() -> bool {
736            true
737        }
738    }
739
740    pub mod meta_store_config {
741        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
742        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
743        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
744        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
745        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
746
747        pub fn max_connections() -> u32 {
748            DEFAULT_MAX_CONNECTIONS
749        }
750
751        pub fn min_connections() -> u32 {
752            DEFAULT_MIN_CONNECTIONS
753        }
754
755        pub fn connection_timeout_sec() -> u64 {
756            DEFAULT_CONNECTION_TIMEOUT_SEC
757        }
758
759        pub fn idle_timeout_sec() -> u64 {
760            DEFAULT_IDLE_TIMEOUT_SEC
761        }
762
763        pub fn acquire_timeout_sec() -> u64 {
764            DEFAULT_ACQUIRE_TIMEOUT_SEC
765        }
766    }
767
768    pub mod compaction_config {
769        const MB: u64 = 1024 * 1024;
770        const GB: u64 = 1024 * 1024 * 1024;
771        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
772        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
773        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
774
775        // decrease this configure when the generation of checkpoint barrier is not frequent.
776        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
777        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
778        // 32MB
779        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
780        const DEFAULT_LEVEL_MULTIPLIER: u64 = 5;
781        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
782        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 300;
783        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
784        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
785        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
786        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
787        const DEFAULT_EMERGENCY_PICKER: bool = true;
788        const DEFAULT_MAX_LEVEL: u32 = 6;
789        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
790        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
791        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 64;
792        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
793        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
794        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 10000; // 10000 * 32M = 320G
795        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
796
797        use crate::catalog::hummock::CompactionFilterFlag;
798
799        pub fn max_bytes_for_level_base() -> u64 {
800            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
801        }
802
803        pub fn max_bytes_for_level_multiplier() -> u64 {
804            DEFAULT_LEVEL_MULTIPLIER
805        }
806
807        pub fn max_compaction_bytes() -> u64 {
808            DEFAULT_MAX_COMPACTION_BYTES
809        }
810
811        pub fn sub_level_max_compaction_bytes() -> u64 {
812            DEFAULT_MIN_COMPACTION_BYTES
813        }
814
815        pub fn level0_tier_compact_file_number() -> u64 {
816            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
817        }
818
819        pub fn target_file_size_base() -> u64 {
820            DEFAULT_TARGET_FILE_SIZE_BASE
821        }
822
823        pub fn compaction_filter_mask() -> u32 {
824            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
825        }
826
827        pub fn max_sub_compaction() -> u32 {
828            DEFAULT_MAX_SUB_COMPACTION
829        }
830
831        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
832            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
833        }
834
835        pub fn level0_sub_level_compact_level_count() -> u32 {
836            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
837        }
838
839        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
840            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
841        }
842
843        pub fn max_space_reclaim_bytes() -> u64 {
844            DEFAULT_MAX_SPACE_RECLAIM_BYTES
845        }
846
847        pub fn level0_max_compact_file_number() -> u64 {
848            DEFAULT_MAX_COMPACTION_FILE_COUNT
849        }
850
851        pub fn tombstone_reclaim_ratio() -> u32 {
852            DEFAULT_TOMBSTONE_RATIO_PERCENT
853        }
854
855        pub fn enable_emergency_picker() -> bool {
856            DEFAULT_EMERGENCY_PICKER
857        }
858
859        pub fn max_level() -> u32 {
860            DEFAULT_MAX_LEVEL
861        }
862
863        pub fn max_l0_compact_level_count() -> u32 {
864            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
865        }
866
867        pub fn sst_allowed_trivial_move_min_size() -> u64 {
868            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
869        }
870
871        pub fn disable_auto_group_scheduling() -> bool {
872            false
873        }
874
875        pub fn max_overlapping_level_size() -> u64 {
876            256 * MB
877        }
878
879        pub fn sst_allowed_trivial_move_max_count() -> u32 {
880            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
881        }
882
883        pub fn emergency_level0_sst_file_count() -> u32 {
884            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
885        }
886
887        pub fn emergency_level0_sub_level_partition() -> u32 {
888            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
889        }
890
891        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
892            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
893        }
894
895        pub fn level0_stop_write_threshold_max_size() -> u64 {
896            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
897        }
898
899        pub fn enable_optimize_l0_interval_selection() -> bool {
900            false
901        }
902    }
903}
risingwave_common/config/meta.rs

risingwave_common/config/
meta.rs
