risingwave_common/config/
meta.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common_proc_macro::serde_prefix_all;
16
17use super::*;
18
19#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
20pub enum MetaBackend {
21    #[default]
22    Mem,
23    Sql, // any database url
24    Sqlite,
25    Postgres,
26    Mysql,
27}
28
29#[derive(Copy, Clone, Debug, Default)]
30pub enum DefaultParallelism {
31    #[default]
32    Full,
33    Default(NonZeroUsize),
34}
35
36impl Serialize for DefaultParallelism {
37    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
38    where
39        S: Serializer,
40    {
41        #[derive(Debug, Serialize, Deserialize)]
42        #[serde(untagged)]
43        enum Parallelism {
44            Str(String),
45            Int(usize),
46        }
47        match self {
48            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
49            DefaultParallelism::Default(val) => {
50                Parallelism::Int(val.get() as _).serialize(serializer)
51            }
52        }
53    }
54}
55
56impl<'de> Deserialize<'de> for DefaultParallelism {
57    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
58    where
59        D: serde::Deserializer<'de>,
60    {
61        #[derive(Debug, Deserialize)]
62        #[serde(untagged)]
63        enum Parallelism {
64            Str(String),
65            Int(usize),
66        }
67        let p = Parallelism::deserialize(deserializer)?;
68        match p {
69            Parallelism::Str(s) => {
70                if s.trim().eq_ignore_ascii_case("full") {
71                    Ok(DefaultParallelism::Full)
72                } else {
73                    Err(serde::de::Error::custom(format!(
74                        "invalid default parallelism: {}",
75                        s
76                    )))
77                }
78            }
79            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
80                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
81                // here because it requires extra context. The check will be done when scheduling jobs.
82                NonZeroUsize::new(i).ok_or_else(|| {
83                    serde::de::Error::custom("default parallelism should not be 0")
84                })?,
85            )),
86        }
87    }
88}
89
90/// The section `[meta]` in `risingwave.toml`.
91#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
92pub struct MetaConfig {
93    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
94    /// are dangling.
95    #[serde(default = "default::meta::min_sst_retention_time_sec")]
96    pub min_sst_retention_time_sec: u64,
97
98    /// Interval of automatic hummock full GC.
99    #[serde(default = "default::meta::full_gc_interval_sec")]
100    pub full_gc_interval_sec: u64,
101
102    /// Max number of object per full GC job can fetch.
103    #[serde(default = "default::meta::full_gc_object_limit")]
104    pub full_gc_object_limit: u64,
105
106    /// Duration in seconds to retain garbage collection history data.
107    #[serde(default = "default::meta::gc_history_retention_time_sec")]
108    pub gc_history_retention_time_sec: u64,
109
110    /// Max number of inflight time travel query.
111    #[serde(default = "default::meta::max_inflight_time_travel_query")]
112    pub max_inflight_time_travel_query: u64,
113
114    /// Schedule compaction for all compaction groups with this interval.
115    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
116    pub periodic_compaction_interval_sec: u64,
117
118    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
119    /// from object store.
120    #[serde(default = "default::meta::vacuum_interval_sec")]
121    pub vacuum_interval_sec: u64,
122
123    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
124    /// meta node.
125    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
126    pub vacuum_spin_interval_ms: u64,
127
128    /// Interval of invoking iceberg garbage collection, to expire old snapshots.
129    #[serde(default = "default::meta::iceberg_gc_interval_sec")]
130    pub iceberg_gc_interval_sec: u64,
131
132    /// Interval of hummock version checkpoint.
133    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
134    pub hummock_version_checkpoint_interval_sec: u64,
135
136    /// If enabled, `SSTable` object file and version delta will be retained.
137    ///
138    /// `SSTable` object file need to be deleted via full GC.
139    ///
140    /// version delta need to be manually deleted.
141    #[serde(default = "default::meta::enable_hummock_data_archive")]
142    pub enable_hummock_data_archive: bool,
143
144    /// The interval at which a Hummock version snapshot is taken for time travel.
145    ///
146    /// Larger value indicates less storage overhead but worse query performance.
147    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
148    pub hummock_time_travel_snapshot_interval: u64,
149
150    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
151    /// attempt is rejected.
152    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
153    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
154
155    /// Maximum allowed heartbeat interval in seconds.
156    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
157    pub max_heartbeat_interval_secs: u32,
158
159    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
160    #[serde(default)]
161    pub disable_recovery: bool,
162
163    /// Whether meta should request pausing all data sources on the next bootstrap.
164    /// This allows us to pause the cluster on next bootstrap in an offline way.
165    /// It's important for standalone or single node deployments.
166    /// In those cases, meta node, frontend and compute may all be co-located.
167    /// If the compute node enters an inconsistent state, and continuously crashloops,
168    /// we may not be able to connect to the cluster to run `alter system set pause_on_next_bootstrap = true;`.
169    /// By providing it in the static config, we can have an offline way to trigger the pause on bootstrap.
170    #[serde(default = "default::meta::pause_on_next_bootstrap_offline")]
171    pub pause_on_next_bootstrap_offline: bool,
172
173    /// Whether to disable adaptive-scaling feature.
174    #[serde(default)]
175    pub disable_automatic_parallelism_control: bool,
176
177    /// The number of streaming jobs per scaling operation.
178    #[serde(default = "default::meta::parallelism_control_batch_size")]
179    pub parallelism_control_batch_size: usize,
180
181    /// The period of parallelism control trigger.
182    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
183    pub parallelism_control_trigger_period_sec: u64,
184
185    /// The first delay of parallelism control.
186    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
187    pub parallelism_control_trigger_first_delay_sec: u64,
188
189    #[serde(default = "default::meta::meta_leader_lease_secs")]
190    pub meta_leader_lease_secs: u64,
191
192    /// After specified seconds of idle (no mview or flush), the process will be exited.
193    /// It is mainly useful for playgrounds.
194    #[serde(default)]
195    pub dangerous_max_idle_secs: Option<u64>,
196
197    /// The default global parallelism for all streaming jobs, if user doesn't specify the
198    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
199    /// otherwise it's a number.
200    #[serde(default = "default::meta::default_parallelism")]
201    pub default_parallelism: DefaultParallelism,
202
203    /// Whether to enable deterministic compaction scheduling, which
204    /// will disable all auto scheduling of compaction tasks.
205    /// Should only be used in e2e tests.
206    #[serde(default)]
207    pub enable_compaction_deterministic: bool,
208
209    /// Enable sanity check when SSTs are committed.
210    #[serde(default)]
211    pub enable_committed_sst_sanity_check: bool,
212
213    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
214    pub node_num_monitor_interval_sec: u64,
215
216    #[serde(default = "default::meta::backend")]
217    pub backend: MetaBackend,
218
219    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
220    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
221    pub periodic_space_reclaim_compaction_interval_sec: u64,
222
223    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
224    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
225    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
226
227    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
228    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
229
230    #[serde(default = "default::meta::move_table_size_limit")]
231    #[deprecated]
232    pub move_table_size_limit: u64,
233
234    #[serde(default = "default::meta::split_group_size_limit")]
235    #[deprecated]
236    pub split_group_size_limit: u64,
237
238    #[serde(default = "default::meta::cut_table_size_limit")]
239    #[deprecated]
240    pub cut_table_size_limit: u64,
241
242    /// Whether to protect dropping a table with incoming sink.
243    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
244    pub protect_drop_table_with_incoming_sink: bool,
245
246    #[serde(default, flatten)]
247    #[config_doc(omitted)]
248    pub unrecognized: Unrecognized<Self>,
249
250    /// Whether config object storage bucket lifecycle to purge stale data.
251    #[serde(default)]
252    pub do_not_config_object_storage_lifecycle: bool,
253
254    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
255    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
256    #[serde(default = "default::meta::partition_vnode_count")]
257    pub partition_vnode_count: u32,
258
259    /// The threshold of write throughput to trigger a group split.
260    #[serde(
261        default = "default::meta::table_high_write_throughput_threshold",
262        alias = "table_write_throughput_threshold"
263    )]
264    pub table_high_write_throughput_threshold: u64,
265
266    #[serde(
267        default = "default::meta::table_low_write_throughput_threshold",
268        alias = "min_table_split_write_throughput"
269    )]
270    /// The threshold of write throughput to trigger a group merge.
271    pub table_low_write_throughput_threshold: u64,
272
273    // If the compaction task does not report heartbeat beyond the
274    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
275    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
276    pub compaction_task_max_heartbeat_interval_secs: u64,
277
278    // If the compaction task does not change in progress beyond the
279    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
280    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
281    pub compaction_task_max_progress_interval_secs: u64,
282
283    #[serde(default)]
284    #[config_doc(nested)]
285    pub compaction_config: CompactionConfig,
286
287    /// Count of partitions of tables in default group and materialized view group.
288    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
289    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
290    /// Set it zero to disable this feature.
291    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
292    pub hybrid_partition_vnode_count: u32,
293
294    #[serde(default = "default::meta::event_log_enabled")]
295    pub event_log_enabled: bool,
296    /// Keeps the latest N events per channel.
297    #[serde(default = "default::meta::event_log_channel_max_size")]
298    pub event_log_channel_max_size: u32,
299
300    #[serde(default)]
301    #[config_doc(omitted)]
302    pub developer: MetaDeveloperConfig,
303    /// Whether compactor should rewrite row to remove dropped column.
304    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
305    pub enable_dropped_column_reclaim: bool,
306
307    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
308    #[serde(default = "default::meta::split_group_size_ratio")]
309    pub split_group_size_ratio: f64,
310
311    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
312    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
313    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
314    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
315    pub table_stat_high_write_throughput_ratio_for_split: f64,
316
317    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
318    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
319    pub table_stat_low_write_throughput_ratio_for_merge: f64,
320
321    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
322    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
323    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
324    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
325    /// The window seconds of table throughput statistic history for split compaction group.
326    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
327    pub table_stat_throuput_window_seconds_for_split: usize,
328
329    /// The window seconds of table throughput statistic history for merge compaction group.
330    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
331    pub table_stat_throuput_window_seconds_for_merge: usize,
332
333    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
334    /// Set it max value of 64-bit number to disable this feature.
335    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
336    pub compact_task_table_size_partition_threshold_low: u64,
337
338    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
339    /// Set it max value of 64-bit number to disable this feature.
340    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
341    pub compact_task_table_size_partition_threshold_high: u64,
342
343    /// The interval of the periodic scheduling compaction group split job.
344    #[serde(
345        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
346        alias = "periodic_split_compact_group_interval_sec"
347    )]
348    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
349
350    /// The interval of the periodic scheduling compaction group merge job.
351    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
352    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
353
354    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
355    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
356    pub compaction_group_merge_dimension_threshold: f64,
357
358    /// The interval that the CDC table splits initialization should yield to avoid overloading upstream system.
359    #[serde(default = "default::meta::cdc_table_split_init_sleep_interval_splits")]
360    pub cdc_table_split_init_sleep_interval_splits: u64,
361
362    /// The duration that the CDC table splits initialization should yield to avoid overloading upstream system.
363    #[serde(default = "default::meta::cdc_table_split_init_sleep_duration_millis")]
364    pub cdc_table_split_init_sleep_duration_millis: u64,
365
366    /// The batch size that the CDC table splits initialization should use when persisting to meta store.
367    #[serde(default = "default::meta::cdc_table_split_init_insert_batch_size")]
368    pub cdc_table_split_init_insert_batch_size: u64,
369
370    /// Whether to automatically migrate legacy table fragments when meta starts.
371    #[serde(default = "default::meta::enable_legacy_table_migration")]
372    pub enable_legacy_table_migration: bool,
373
374    #[serde(default)]
375    #[config_doc(nested)]
376    pub meta_store_config: MetaStoreConfig,
377}
378
379/// Note: only applies to meta store backends other than `SQLite`.
380#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
381pub struct MetaStoreConfig {
382    /// Maximum number of connections for the meta store connection pool.
383    #[serde(default = "default::meta_store_config::max_connections")]
384    pub max_connections: u32,
385    /// Minimum number of connections for the meta store connection pool.
386    #[serde(default = "default::meta_store_config::min_connections")]
387    pub min_connections: u32,
388    /// Connection timeout in seconds for a meta store connection.
389    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
390    pub connection_timeout_sec: u64,
391    /// Idle timeout in seconds for a meta store connection.
392    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
393    pub idle_timeout_sec: u64,
394    /// Acquire timeout in seconds for a meta store connection.
395    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
396    pub acquire_timeout_sec: u64,
397}
398
399/// The subsections `[meta.developer]`.
400///
401/// It is put at [`MetaConfig::developer`].
402#[serde_prefix_all("meta_", mode = "alias")]
403#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
404pub struct MetaDeveloperConfig {
405    /// The number of traces to be cached in-memory by the tracing collector
406    /// embedded in the meta node.
407    #[serde(default = "default::developer::meta_cached_traces_num")]
408    pub cached_traces_num: u32,
409
410    /// The maximum memory usage in bytes for the tracing collector embedded
411    /// in the meta node.
412    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
413    pub cached_traces_memory_limit_bytes: usize,
414
415    /// Compaction picker config
416    #[serde(default = "default::developer::enable_trivial_move")]
417    pub enable_trivial_move: bool,
418    #[serde(default = "default::developer::enable_check_task_level_overlap")]
419    pub enable_check_task_level_overlap: bool,
420    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
421    pub max_trivial_move_task_count_per_loop: usize,
422
423    #[serde(default = "default::developer::max_get_task_probe_times")]
424    pub max_get_task_probe_times: usize,
425
426    /// Max number of actor allowed per parallelism (default = 100).
427    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
428    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
429    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
430
431    /// Max number of actor allowed per parallelism (default = 400).
432    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
433    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
434    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
435
436    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
437    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
438    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
439
440    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
441    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
442    pub hummock_time_travel_sst_info_insert_batch_size: usize,
443
444    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
445    pub time_travel_vacuum_interval_sec: u64,
446
447    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
448    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
449    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
450
451    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
452    pub hummock_gc_history_insert_batch_size: usize,
453
454    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
455    pub hummock_time_travel_filter_out_objects_batch_size: usize,
456
457    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
458    pub hummock_time_travel_filter_out_objects_v1: bool,
459
460    #[serde(
461        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
462    )]
463    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
464
465    #[serde(
466        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
467    )]
468    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
469
470    #[serde(default)]
471    pub compute_client_config: RpcClientConfig,
472
473    #[serde(default)]
474    pub stream_client_config: RpcClientConfig,
475
476    #[serde(default)]
477    pub frontend_client_config: RpcClientConfig,
478}
479
480#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
481pub struct CompactionConfig {
482    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
483    pub max_bytes_for_level_base: u64,
484    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
485    pub max_bytes_for_level_multiplier: u64,
486    #[serde(default = "default::compaction_config::max_compaction_bytes")]
487    pub max_compaction_bytes: u64,
488    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
489    pub sub_level_max_compaction_bytes: u64,
490    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
491    pub level0_tier_compact_file_number: u64,
492    #[serde(default = "default::compaction_config::target_file_size_base")]
493    pub target_file_size_base: u64,
494    #[serde(default = "default::compaction_config::compaction_filter_mask")]
495    pub compaction_filter_mask: u32,
496    #[serde(default = "default::compaction_config::max_sub_compaction")]
497    pub max_sub_compaction: u32,
498    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
499    pub level0_stop_write_threshold_sub_level_number: u64,
500    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
501    pub level0_sub_level_compact_level_count: u32,
502    #[serde(
503        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
504    )]
505    pub level0_overlapping_sub_level_compact_level_count: u32,
506    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
507    pub max_space_reclaim_bytes: u64,
508    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
509    pub level0_max_compact_file_number: u64,
510    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
511    pub tombstone_reclaim_ratio: u32,
512    #[serde(default = "default::compaction_config::enable_emergency_picker")]
513    pub enable_emergency_picker: bool,
514    #[serde(default = "default::compaction_config::max_level")]
515    pub max_level: u32,
516    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
517    pub sst_allowed_trivial_move_min_size: u64,
518    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
519    pub sst_allowed_trivial_move_max_count: u32,
520    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
521    pub max_l0_compact_level_count: u32,
522    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
523    pub disable_auto_group_scheduling: bool,
524    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
525    pub max_overlapping_level_size: u64,
526    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
527    pub emergency_level0_sst_file_count: u32,
528    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
529    pub emergency_level0_sub_level_partition: u32,
530    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
531    pub level0_stop_write_threshold_max_sst_count: u32,
532    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
533    pub level0_stop_write_threshold_max_size: u64,
534    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
535    pub enable_optimize_l0_interval_selection: bool,
536    #[serde(default = "default::compaction_config::vnode_aligned_level_size_threshold")]
537    pub vnode_aligned_level_size_threshold: Option<u64>,
538    #[serde(default = "default::compaction_config::max_kv_count_for_xor16")]
539    pub max_kv_count_for_xor16: Option<u64>,
540}
541
542pub mod default {
543    pub use crate::config::default::developer;
544
545    pub mod meta {
546        use crate::config::{DefaultParallelism, MetaBackend};
547
548        pub fn min_sst_retention_time_sec() -> u64 {
549            3600 * 6
550        }
551
552        pub fn gc_history_retention_time_sec() -> u64 {
553            3600 * 6
554        }
555
556        pub fn full_gc_interval_sec() -> u64 {
557            3600
558        }
559
560        pub fn full_gc_object_limit() -> u64 {
561            100_000
562        }
563
564        pub fn max_inflight_time_travel_query() -> u64 {
565            1000
566        }
567
568        pub fn periodic_compaction_interval_sec() -> u64 {
569            60
570        }
571
572        pub fn vacuum_interval_sec() -> u64 {
573            30
574        }
575
576        pub fn vacuum_spin_interval_ms() -> u64 {
577            100
578        }
579
580        pub fn iceberg_gc_interval_sec() -> u64 {
581            3600
582        }
583
584        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
585            30
586        }
587
588        pub fn enable_hummock_data_archive() -> bool {
589            false
590        }
591
592        pub fn hummock_time_travel_snapshot_interval() -> u64 {
593            100
594        }
595
596        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
597            10
598        }
599
600        pub fn max_heartbeat_interval_sec() -> u32 {
601            60
602        }
603
604        pub fn meta_leader_lease_secs() -> u64 {
605            30
606        }
607
608        pub fn default_parallelism() -> DefaultParallelism {
609            DefaultParallelism::Full
610        }
611
612        pub fn pause_on_next_bootstrap_offline() -> bool {
613            false
614        }
615
616        pub fn node_num_monitor_interval_sec() -> u64 {
617            10
618        }
619
620        pub fn backend() -> MetaBackend {
621            MetaBackend::Mem
622        }
623
624        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
625            3600 // 60min
626        }
627
628        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
629            1800 // 30mi
630        }
631
632        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
633            10 // 10s
634        }
635
636        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
637            600
638        }
639
640        // limit the size of state table to trigger split by high throughput
641        pub fn move_table_size_limit() -> u64 {
642            10 * 1024 * 1024 * 1024 // 10GB
643        }
644
645        // limit the size of group to trigger split by group_size and avoid too many small groups
646        pub fn split_group_size_limit() -> u64 {
647            64 * 1024 * 1024 * 1024 // 64GB
648        }
649
650        pub fn protect_drop_table_with_incoming_sink() -> bool {
651            false
652        }
653
654        pub fn partition_vnode_count() -> u32 {
655            16
656        }
657
658        pub fn table_high_write_throughput_threshold() -> u64 {
659            16 * 1024 * 1024 // 16MB
660        }
661
662        pub fn table_low_write_throughput_threshold() -> u64 {
663            4 * 1024 * 1024 // 4MB
664        }
665
666        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
667            30 // 30s
668        }
669
670        pub fn compaction_task_max_progress_interval_secs() -> u64 {
671            60 * 10 // 10min
672        }
673
674        pub fn cut_table_size_limit() -> u64 {
675            1024 * 1024 * 1024 // 1GB
676        }
677
678        pub fn hybrid_partition_vnode_count() -> u32 {
679            4
680        }
681
682        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
683            128 * 1024 * 1024 // 128MB
684        }
685
686        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
687            512 * 1024 * 1024 // 512MB
688        }
689
690        pub fn event_log_enabled() -> bool {
691            true
692        }
693
694        pub fn event_log_channel_max_size() -> u32 {
695            10
696        }
697
698        pub fn parallelism_control_batch_size() -> usize {
699            10
700        }
701
702        pub fn parallelism_control_trigger_period_sec() -> u64 {
703            10
704        }
705
706        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
707            30
708        }
709
710        pub fn enable_dropped_column_reclaim() -> bool {
711            false
712        }
713
714        pub fn split_group_size_ratio() -> f64 {
715            0.9
716        }
717
718        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
719            0.5
720        }
721
722        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
723            0.7
724        }
725
726        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
727            60
728        }
729
730        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
731            240
732        }
733
734        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
735            60 * 10 // 10min
736        }
737
738        pub fn compaction_group_merge_dimension_threshold() -> f64 {
739            1.2
740        }
741
742        pub fn cdc_table_split_init_sleep_interval_splits() -> u64 {
743            1000
744        }
745
746        pub fn cdc_table_split_init_sleep_duration_millis() -> u64 {
747            500
748        }
749
750        pub fn cdc_table_split_init_insert_batch_size() -> u64 {
751            100
752        }
753
754        pub fn enable_legacy_table_migration() -> bool {
755            true
756        }
757    }
758
759    pub mod meta_store_config {
760        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
761        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
762        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
763        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
764        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
765
766        pub fn max_connections() -> u32 {
767            DEFAULT_MAX_CONNECTIONS
768        }
769
770        pub fn min_connections() -> u32 {
771            DEFAULT_MIN_CONNECTIONS
772        }
773
774        pub fn connection_timeout_sec() -> u64 {
775            DEFAULT_CONNECTION_TIMEOUT_SEC
776        }
777
778        pub fn idle_timeout_sec() -> u64 {
779            DEFAULT_IDLE_TIMEOUT_SEC
780        }
781
782        pub fn acquire_timeout_sec() -> u64 {
783            DEFAULT_ACQUIRE_TIMEOUT_SEC
784        }
785    }
786
787    pub mod compaction_config {
788        const MB: u64 = 1024 * 1024;
789        const GB: u64 = 1024 * 1024 * 1024;
790        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
791        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
792        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
793
794        // decrease this configure when the generation of checkpoint barrier is not frequent.
795        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
796        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
797        // 32MB
798        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
799        const DEFAULT_LEVEL_MULTIPLIER: u64 = 10;
800        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
801        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 128;
802        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
803        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
804        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
805        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
806        const DEFAULT_EMERGENCY_PICKER: bool = true;
807        const DEFAULT_MAX_LEVEL: u32 = 6;
808        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
809        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
810        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 256;
811        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
812        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
813        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 5000;
814        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
815        const DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION: bool = true;
816        const DEFAULT_VNODE_ALIGNED_LEVEL_SIZE_THRESHOLD: Option<u64> = None;
817        pub const DEFAULT_MAX_KV_COUNT_FOR_XOR16: u64 = 256 * 1024;
818
819        use crate::catalog::hummock::CompactionFilterFlag;
820
821        pub fn max_bytes_for_level_base() -> u64 {
822            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
823        }
824
825        pub fn max_bytes_for_level_multiplier() -> u64 {
826            DEFAULT_LEVEL_MULTIPLIER
827        }
828
829        pub fn max_compaction_bytes() -> u64 {
830            DEFAULT_MAX_COMPACTION_BYTES
831        }
832
833        pub fn sub_level_max_compaction_bytes() -> u64 {
834            DEFAULT_MIN_COMPACTION_BYTES
835        }
836
837        pub fn level0_tier_compact_file_number() -> u64 {
838            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
839        }
840
841        pub fn target_file_size_base() -> u64 {
842            DEFAULT_TARGET_FILE_SIZE_BASE
843        }
844
845        pub fn compaction_filter_mask() -> u32 {
846            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
847        }
848
849        pub fn max_sub_compaction() -> u32 {
850            DEFAULT_MAX_SUB_COMPACTION
851        }
852
853        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
854            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
855        }
856
857        pub fn level0_sub_level_compact_level_count() -> u32 {
858            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
859        }
860
861        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
862            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
863        }
864
865        pub fn max_space_reclaim_bytes() -> u64 {
866            DEFAULT_MAX_SPACE_RECLAIM_BYTES
867        }
868
869        pub fn level0_max_compact_file_number() -> u64 {
870            DEFAULT_MAX_COMPACTION_FILE_COUNT
871        }
872
873        pub fn tombstone_reclaim_ratio() -> u32 {
874            DEFAULT_TOMBSTONE_RATIO_PERCENT
875        }
876
877        pub fn enable_emergency_picker() -> bool {
878            DEFAULT_EMERGENCY_PICKER
879        }
880
881        pub fn max_level() -> u32 {
882            DEFAULT_MAX_LEVEL
883        }
884
885        pub fn max_l0_compact_level_count() -> u32 {
886            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
887        }
888
889        pub fn sst_allowed_trivial_move_min_size() -> u64 {
890            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
891        }
892
893        pub fn disable_auto_group_scheduling() -> bool {
894            false
895        }
896
897        pub fn max_overlapping_level_size() -> u64 {
898            256 * MB
899        }
900
901        pub fn sst_allowed_trivial_move_max_count() -> u32 {
902            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
903        }
904
905        pub fn emergency_level0_sst_file_count() -> u32 {
906            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
907        }
908
909        pub fn emergency_level0_sub_level_partition() -> u32 {
910            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
911        }
912
913        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
914            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
915        }
916
917        pub fn level0_stop_write_threshold_max_size() -> u64 {
918            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
919        }
920
921        pub fn enable_optimize_l0_interval_selection() -> bool {
922            DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION
923        }
924
925        pub fn vnode_aligned_level_size_threshold() -> Option<u64> {
926            DEFAULT_VNODE_ALIGNED_LEVEL_SIZE_THRESHOLD
927        }
928
929        pub fn max_kv_count_for_xor16() -> Option<u64> {
930            Some(DEFAULT_MAX_KV_COUNT_FOR_XOR16)
931        }
932    }
933}
risingwave_common/config/meta.rs

risingwave_common/config/
meta.rs
