risingwave_common/config/
meta.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common_proc_macro::serde_prefix_all;
16
17use super::*;
18
19#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
20pub enum MetaBackend {
21    #[default]
22    Mem,
23    Sql, // any database url
24    Sqlite,
25    Postgres,
26    Mysql,
27}
28
29#[derive(Copy, Clone, Debug, Default)]
30pub enum DefaultParallelism {
31    #[default]
32    Full,
33    Default(NonZeroUsize),
34}
35
36impl Serialize for DefaultParallelism {
37    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
38    where
39        S: Serializer,
40    {
41        #[derive(Debug, Serialize, Deserialize)]
42        #[serde(untagged)]
43        enum Parallelism {
44            Str(String),
45            Int(usize),
46        }
47        match self {
48            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
49            DefaultParallelism::Default(val) => {
50                Parallelism::Int(val.get() as _).serialize(serializer)
51            }
52        }
53    }
54}
55
56impl<'de> Deserialize<'de> for DefaultParallelism {
57    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
58    where
59        D: serde::Deserializer<'de>,
60    {
61        #[derive(Debug, Deserialize)]
62        #[serde(untagged)]
63        enum Parallelism {
64            Str(String),
65            Int(usize),
66        }
67        let p = Parallelism::deserialize(deserializer)?;
68        match p {
69            Parallelism::Str(s) => {
70                if s.trim().eq_ignore_ascii_case("full") {
71                    Ok(DefaultParallelism::Full)
72                } else {
73                    Err(serde::de::Error::custom(format!(
74                        "invalid default parallelism: {}",
75                        s
76                    )))
77                }
78            }
79            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
80                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
81                // here because it requires extra context. The check will be done when scheduling jobs.
82                NonZeroUsize::new(i).ok_or_else(|| {
83                    serde::de::Error::custom("default parallelism should not be 0")
84                })?,
85            )),
86        }
87    }
88}
89
90/// The section `[meta]` in `risingwave.toml`.
91#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
92#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
93pub struct MetaConfig {
94    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
95    /// are dangling.
96    #[serde(default = "default::meta::min_sst_retention_time_sec")]
97    pub min_sst_retention_time_sec: u64,
98
99    /// Interval of automatic hummock full GC.
100    #[serde(default = "default::meta::full_gc_interval_sec")]
101    pub full_gc_interval_sec: u64,
102
103    /// Max number of object per full GC job can fetch.
104    #[serde(default = "default::meta::full_gc_object_limit")]
105    pub full_gc_object_limit: u64,
106
107    /// Duration in seconds to retain garbage collection history data.
108    #[serde(default = "default::meta::gc_history_retention_time_sec")]
109    pub gc_history_retention_time_sec: u64,
110
111    /// Max number of inflight time travel query.
112    #[serde(default = "default::meta::max_inflight_time_travel_query")]
113    pub max_inflight_time_travel_query: u64,
114
115    /// Schedule `Dynamic` compaction for all compaction groups with this interval.
116    /// Groups in cooldown (recently found to have no compaction work) are skipped.
117    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
118    pub periodic_compaction_interval_sec: u64,
119
120    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
121    /// from object store.
122    #[serde(default = "default::meta::vacuum_interval_sec")]
123    pub vacuum_interval_sec: u64,
124
125    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
126    /// meta node.
127    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
128    pub vacuum_spin_interval_ms: u64,
129
130    /// Interval of invoking iceberg garbage collection, to expire old snapshots.
131    #[serde(default = "default::meta::iceberg_gc_interval_sec")]
132    pub iceberg_gc_interval_sec: u64,
133
134    /// Interval of hummock version checkpoint.
135    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
136    pub hummock_version_checkpoint_interval_sec: u64,
137
138    /// If enabled, `SSTable` object file and version delta will be retained.
139    ///
140    /// `SSTable` object file need to be deleted via full GC.
141    ///
142    /// version delta need to be manually deleted.
143    #[serde(default = "default::meta::enable_hummock_data_archive")]
144    pub enable_hummock_data_archive: bool,
145
146    /// The interval at which a Hummock version snapshot is taken for time travel.
147    ///
148    /// Larger value indicates less storage overhead but worse query performance.
149    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
150    pub hummock_time_travel_snapshot_interval: u64,
151
152    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
153    /// attempt is rejected.
154    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
155    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
156
157    /// Maximum allowed heartbeat interval in seconds.
158    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
159    pub max_heartbeat_interval_secs: u32,
160
161    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
162    #[serde(default)]
163    pub disable_recovery: bool,
164
165    /// Whether meta should request pausing all data sources on the next bootstrap.
166    /// This allows us to pause the cluster on next bootstrap in an offline way.
167    /// It's important for standalone or single node deployments.
168    /// In those cases, meta node, frontend and compute may all be co-located.
169    /// If the compute node enters an inconsistent state, and continuously crashloops,
170    /// we may not be able to connect to the cluster to run `alter system set pause_on_next_bootstrap = true;`.
171    /// By providing it in the static config, we can have an offline way to trigger the pause on bootstrap.
172    #[serde(default = "default::meta::pause_on_next_bootstrap_offline")]
173    pub pause_on_next_bootstrap_offline: bool,
174
175    /// Whether to disable adaptive-scaling feature.
176    #[serde(default)]
177    pub disable_automatic_parallelism_control: bool,
178
179    /// The number of streaming jobs per scaling operation.
180    #[serde(default = "default::meta::parallelism_control_batch_size")]
181    pub parallelism_control_batch_size: usize,
182
183    /// The period of parallelism control trigger.
184    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
185    pub parallelism_control_trigger_period_sec: u64,
186
187    /// The first delay of parallelism control.
188    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
189    pub parallelism_control_trigger_first_delay_sec: u64,
190
191    #[serde(default = "default::meta::meta_leader_lease_secs")]
192    pub meta_leader_lease_secs: u64,
193
194    /// After specified seconds of idle (no mview or flush), the process will be exited.
195    /// It is mainly useful for playgrounds.
196    #[serde(default)]
197    pub dangerous_max_idle_secs: Option<u64>,
198
199    /// The default global parallelism for all streaming jobs, if user doesn't specify the
200    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
201    /// otherwise it's a number.
202    #[serde(default = "default::meta::default_parallelism")]
203    pub default_parallelism: DefaultParallelism,
204
205    /// Whether to enable deterministic compaction scheduling, which
206    /// will disable all auto scheduling of compaction tasks.
207    /// Should only be used in e2e tests.
208    #[serde(default)]
209    pub enable_compaction_deterministic: bool,
210
211    /// Enable sanity check when SSTs are committed.
212    #[serde(default)]
213    pub enable_committed_sst_sanity_check: bool,
214
215    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
216    pub node_num_monitor_interval_sec: u64,
217
218    #[serde(default = "default::meta::backend")]
219    pub backend: MetaBackend,
220
221    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
222    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
223    pub periodic_space_reclaim_compaction_interval_sec: u64,
224
225    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
226    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
227    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
228
229    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
230    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
231
232    #[serde(default = "default::meta::move_table_size_limit")]
233    #[deprecated]
234    pub move_table_size_limit: u64,
235
236    #[serde(default = "default::meta::split_group_size_limit")]
237    #[deprecated]
238    pub split_group_size_limit: u64,
239
240    #[serde(default = "default::meta::cut_table_size_limit")]
241    #[deprecated]
242    pub cut_table_size_limit: u64,
243
244    /// Whether to protect dropping a table with incoming sink.
245    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
246    pub protect_drop_table_with_incoming_sink: bool,
247
248    #[serde(default, flatten)]
249    #[config_doc(omitted)]
250    pub unrecognized: Unrecognized<Self>,
251
252    /// Whether config object storage bucket lifecycle to purge stale data.
253    #[serde(default)]
254    pub do_not_config_object_storage_lifecycle: bool,
255
256    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
257    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
258    #[serde(default = "default::meta::partition_vnode_count")]
259    pub partition_vnode_count: u32,
260
261    /// The threshold of write throughput to trigger a group split.
262    #[serde(
263        default = "default::meta::table_high_write_throughput_threshold",
264        alias = "table_write_throughput_threshold"
265    )]
266    pub table_high_write_throughput_threshold: u64,
267
268    #[serde(
269        default = "default::meta::table_low_write_throughput_threshold",
270        alias = "min_table_split_write_throughput"
271    )]
272    /// The threshold of write throughput to trigger a group merge.
273    pub table_low_write_throughput_threshold: u64,
274
275    // If the compaction task does not report heartbeat beyond the
276    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
277    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
278    pub compaction_task_max_heartbeat_interval_secs: u64,
279
280    // If the compaction task does not change in progress beyond the
281    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
282    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
283    pub compaction_task_max_progress_interval_secs: u64,
284
285    #[serde(default)]
286    #[config_doc(nested)]
287    pub compaction_config: CompactionConfig,
288
289    /// Count of partitions of tables in default group and materialized view group.
290    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
291    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
292    /// Set it zero to disable this feature.
293    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
294    pub hybrid_partition_vnode_count: u32,
295
296    #[serde(default = "default::meta::event_log_enabled")]
297    pub event_log_enabled: bool,
298    /// Keeps the latest N events per channel.
299    #[serde(default = "default::meta::event_log_channel_max_size")]
300    pub event_log_channel_max_size: u32,
301
302    #[serde(default)]
303    #[config_doc(nested)]
304    pub developer: MetaDeveloperConfig,
305
306    /// Whether compactor should rewrite row to remove dropped column.
307    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
308    pub enable_dropped_column_reclaim: bool,
309
310    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
311    #[serde(default = "default::meta::split_group_size_ratio")]
312    pub split_group_size_ratio: f64,
313
314    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
315    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
316    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
317    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
318    pub table_stat_high_write_throughput_ratio_for_split: f64,
319
320    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
321    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
322    pub table_stat_low_write_throughput_ratio_for_merge: f64,
323
324    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
325    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
326    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
327    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
328    /// The window seconds of table throughput statistic history for split compaction group.
329    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
330    pub table_stat_throuput_window_seconds_for_split: usize,
331
332    /// The window seconds of table throughput statistic history for merge compaction group.
333    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
334    pub table_stat_throuput_window_seconds_for_merge: usize,
335
336    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
337    /// Set it max value of 64-bit number to disable this feature.
338    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
339    pub compact_task_table_size_partition_threshold_low: u64,
340
341    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
342    /// Set it max value of 64-bit number to disable this feature.
343    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
344    pub compact_task_table_size_partition_threshold_high: u64,
345
346    /// The interval of the periodic scheduling compaction group split job.
347    #[serde(
348        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
349        alias = "periodic_split_compact_group_interval_sec"
350    )]
351    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
352
353    /// The interval of the periodic scheduling compaction group merge job.
354    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
355    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
356
357    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
358    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
359    pub compaction_group_merge_dimension_threshold: f64,
360
361    /// The interval that the CDC table splits initialization should yield to avoid overloading upstream system.
362    #[serde(default = "default::meta::cdc_table_split_init_sleep_interval_splits")]
363    pub cdc_table_split_init_sleep_interval_splits: u64,
364
365    /// The duration that the CDC table splits initialization should yield to avoid overloading upstream system.
366    #[serde(default = "default::meta::cdc_table_split_init_sleep_duration_millis")]
367    pub cdc_table_split_init_sleep_duration_millis: u64,
368
369    /// The batch size that the CDC table splits initialization should use when persisting to meta store.
370    #[serde(default = "default::meta::cdc_table_split_init_insert_batch_size")]
371    pub cdc_table_split_init_insert_batch_size: u64,
372
373    /// Whether to automatically migrate legacy table fragments when meta starts.
374    #[serde(default = "default::meta::enable_legacy_table_migration")]
375    pub enable_legacy_table_migration: bool,
376
377    #[serde(default)]
378    #[config_doc(nested)]
379    pub meta_store_config: MetaStoreConfig,
380}
381
382/// Note: only applies to meta store backends other than `SQLite`.
383#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
384#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
385pub struct MetaStoreConfig {
386    /// Maximum number of connections for the meta store connection pool.
387    #[serde(default = "default::meta_store_config::max_connections")]
388    pub max_connections: u32,
389    /// Minimum number of connections for the meta store connection pool.
390    #[serde(default = "default::meta_store_config::min_connections")]
391    pub min_connections: u32,
392    /// Connection timeout in seconds for a meta store connection.
393    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
394    pub connection_timeout_sec: u64,
395    /// Idle timeout in seconds for a meta store connection.
396    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
397    pub idle_timeout_sec: u64,
398    /// Acquire timeout in seconds for a meta store connection.
399    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
400    pub acquire_timeout_sec: u64,
401}
402
403/// The subsections `[meta.developer]`.
404///
405/// It is put at [`MetaConfig::developer`].
406#[serde_prefix_all("meta_", mode = "alias")]
407#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
408#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
409pub struct MetaDeveloperConfig {
410    /// The number of traces to be cached in-memory by the tracing collector
411    /// embedded in the meta node.
412    #[serde(default = "default::developer::meta_cached_traces_num")]
413    pub cached_traces_num: u32,
414
415    /// The maximum memory usage in bytes for the tracing collector embedded
416    /// in the meta node.
417    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
418    pub cached_traces_memory_limit_bytes: usize,
419
420    /// Compaction picker config
421    #[serde(default = "default::developer::enable_trivial_move")]
422    pub enable_trivial_move: bool,
423    #[serde(default = "default::developer::enable_check_task_level_overlap")]
424    pub enable_check_task_level_overlap: bool,
425    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
426    pub max_trivial_move_task_count_per_loop: usize,
427
428    #[serde(default = "default::developer::max_get_task_probe_times")]
429    pub max_get_task_probe_times: usize,
430
431    /// Max number of actor allowed per parallelism (default = 100).
432    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
433    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
434    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
435
436    /// Max number of actor allowed per parallelism (default = 400).
437    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
438    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
439    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
440
441    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
442    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
443    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
444
445    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
446    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
447    pub hummock_time_travel_sst_info_insert_batch_size: usize,
448
449    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
450    pub time_travel_vacuum_interval_sec: u64,
451
452    #[serde(default = "default::developer::time_travel_vacuum_max_version_count")]
453    pub time_travel_vacuum_max_version_count: Option<u32>,
454
455    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
456    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
457    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
458
459    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
460    pub hummock_gc_history_insert_batch_size: usize,
461
462    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
463    pub hummock_time_travel_filter_out_objects_batch_size: usize,
464
465    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
466    pub hummock_time_travel_filter_out_objects_v1: bool,
467
468    #[serde(
469        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
470    )]
471    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
472
473    #[serde(
474        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
475    )]
476    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
477
478    #[serde(default)]
479    pub compute_client_config: RpcClientConfig,
480
481    #[serde(default)]
482    pub stream_client_config: RpcClientConfig,
483
484    #[serde(default)]
485    pub frontend_client_config: RpcClientConfig,
486}
487
488#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
489#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
490pub struct CompactionConfig {
491    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
492    pub max_bytes_for_level_base: u64,
493    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
494    pub max_bytes_for_level_multiplier: u64,
495    #[serde(default = "default::compaction_config::max_compaction_bytes")]
496    pub max_compaction_bytes: u64,
497    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
498    pub sub_level_max_compaction_bytes: u64,
499    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
500    pub level0_tier_compact_file_number: u64,
501    #[serde(default = "default::compaction_config::target_file_size_base")]
502    pub target_file_size_base: u64,
503    #[serde(default = "default::compaction_config::compaction_filter_mask")]
504    pub compaction_filter_mask: u32,
505    #[serde(default = "default::compaction_config::max_sub_compaction")]
506    pub max_sub_compaction: u32,
507    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
508    pub level0_stop_write_threshold_sub_level_number: u64,
509    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
510    pub level0_sub_level_compact_level_count: u32,
511    #[serde(
512        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
513    )]
514    pub level0_overlapping_sub_level_compact_level_count: u32,
515    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
516    pub max_space_reclaim_bytes: u64,
517    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
518    pub level0_max_compact_file_number: u64,
519    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
520    pub tombstone_reclaim_ratio: u32,
521    #[serde(default = "default::compaction_config::enable_emergency_picker")]
522    pub enable_emergency_picker: bool,
523    #[serde(default = "default::compaction_config::max_level")]
524    pub max_level: u32,
525    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
526    pub sst_allowed_trivial_move_min_size: u64,
527    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
528    pub sst_allowed_trivial_move_max_count: u32,
529    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
530    pub max_l0_compact_level_count: u32,
531    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
532    pub disable_auto_group_scheduling: bool,
533    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
534    pub max_overlapping_level_size: u64,
535    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
536    pub emergency_level0_sst_file_count: u32,
537    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
538    pub emergency_level0_sub_level_partition: u32,
539    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
540    pub level0_stop_write_threshold_max_sst_count: u32,
541    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
542    pub level0_stop_write_threshold_max_size: u64,
543    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
544    pub enable_optimize_l0_interval_selection: bool,
545    #[serde(default = "default::compaction_config::vnode_aligned_level_size_threshold")]
546    pub vnode_aligned_level_size_threshold: Option<u64>,
547    #[serde(default = "default::compaction_config::max_kv_count_for_xor16")]
548    pub max_kv_count_for_xor16: Option<u64>,
549}
550
551pub mod default {
552    pub use crate::config::default::developer;
553
554    pub mod meta {
555        use crate::config::{DefaultParallelism, MetaBackend};
556
557        pub fn min_sst_retention_time_sec() -> u64 {
558            3600 * 6
559        }
560
561        pub fn gc_history_retention_time_sec() -> u64 {
562            3600 * 6
563        }
564
565        pub fn full_gc_interval_sec() -> u64 {
566            3600
567        }
568
569        pub fn full_gc_object_limit() -> u64 {
570            100_000
571        }
572
573        pub fn max_inflight_time_travel_query() -> u64 {
574            1000
575        }
576
577        pub fn periodic_compaction_interval_sec() -> u64 {
578            300
579        }
580
581        pub fn vacuum_interval_sec() -> u64 {
582            30
583        }
584
585        pub fn vacuum_spin_interval_ms() -> u64 {
586            100
587        }
588
589        pub fn iceberg_gc_interval_sec() -> u64 {
590            3600
591        }
592
593        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
594            30
595        }
596
597        pub fn enable_hummock_data_archive() -> bool {
598            false
599        }
600
601        pub fn hummock_time_travel_snapshot_interval() -> u64 {
602            100
603        }
604
605        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
606            10
607        }
608
609        pub fn max_heartbeat_interval_sec() -> u32 {
610            60
611        }
612
613        pub fn meta_leader_lease_secs() -> u64 {
614            30
615        }
616
617        pub fn default_parallelism() -> DefaultParallelism {
618            DefaultParallelism::Full
619        }
620
621        pub fn pause_on_next_bootstrap_offline() -> bool {
622            false
623        }
624
625        pub fn node_num_monitor_interval_sec() -> u64 {
626            10
627        }
628
629        pub fn backend() -> MetaBackend {
630            MetaBackend::Mem
631        }
632
633        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
634            3600 // 60min
635        }
636
637        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
638            1800 // 30mi
639        }
640
641        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
642            10 // 10s
643        }
644
645        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
646            600
647        }
648
649        // limit the size of state table to trigger split by high throughput
650        pub fn move_table_size_limit() -> u64 {
651            10 * 1024 * 1024 * 1024 // 10GB
652        }
653
654        // limit the size of group to trigger split by group_size and avoid too many small groups
655        pub fn split_group_size_limit() -> u64 {
656            64 * 1024 * 1024 * 1024 // 64GB
657        }
658
659        pub fn protect_drop_table_with_incoming_sink() -> bool {
660            false
661        }
662
663        pub fn partition_vnode_count() -> u32 {
664            16
665        }
666
667        pub fn table_high_write_throughput_threshold() -> u64 {
668            16 * 1024 * 1024 // 16MB
669        }
670
671        pub fn table_low_write_throughput_threshold() -> u64 {
672            4 * 1024 * 1024 // 4MB
673        }
674
675        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
676            30 // 30s
677        }
678
679        pub fn compaction_task_max_progress_interval_secs() -> u64 {
680            60 * 10 // 10min
681        }
682
683        pub fn cut_table_size_limit() -> u64 {
684            1024 * 1024 * 1024 // 1GB
685        }
686
687        pub fn hybrid_partition_vnode_count() -> u32 {
688            4
689        }
690
691        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
692            128 * 1024 * 1024 // 128MB
693        }
694
695        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
696            512 * 1024 * 1024 // 512MB
697        }
698
699        pub fn event_log_enabled() -> bool {
700            true
701        }
702
703        pub fn event_log_channel_max_size() -> u32 {
704            10
705        }
706
707        pub fn parallelism_control_batch_size() -> usize {
708            10
709        }
710
711        pub fn parallelism_control_trigger_period_sec() -> u64 {
712            10
713        }
714
715        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
716            30
717        }
718
719        pub fn enable_dropped_column_reclaim() -> bool {
720            false
721        }
722
723        pub fn split_group_size_ratio() -> f64 {
724            0.9
725        }
726
727        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
728            0.5
729        }
730
731        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
732            0.7
733        }
734
735        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
736            60
737        }
738
739        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
740            240
741        }
742
743        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
744            60 * 10 // 10min
745        }
746
747        pub fn compaction_group_merge_dimension_threshold() -> f64 {
748            1.2
749        }
750
751        pub fn cdc_table_split_init_sleep_interval_splits() -> u64 {
752            1000
753        }
754
755        pub fn cdc_table_split_init_sleep_duration_millis() -> u64 {
756            500
757        }
758
759        pub fn cdc_table_split_init_insert_batch_size() -> u64 {
760            100
761        }
762
763        pub fn enable_legacy_table_migration() -> bool {
764            true
765        }
766    }
767
768    pub mod meta_store_config {
769        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
770        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
771        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
772        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
773        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
774
775        pub fn max_connections() -> u32 {
776            DEFAULT_MAX_CONNECTIONS
777        }
778
779        pub fn min_connections() -> u32 {
780            DEFAULT_MIN_CONNECTIONS
781        }
782
783        pub fn connection_timeout_sec() -> u64 {
784            DEFAULT_CONNECTION_TIMEOUT_SEC
785        }
786
787        pub fn idle_timeout_sec() -> u64 {
788            DEFAULT_IDLE_TIMEOUT_SEC
789        }
790
791        pub fn acquire_timeout_sec() -> u64 {
792            DEFAULT_ACQUIRE_TIMEOUT_SEC
793        }
794    }
795
796    pub mod compaction_config {
797        const MB: u64 = 1024 * 1024;
798        const GB: u64 = 1024 * 1024 * 1024;
799        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
800        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
801        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
802
803        // decrease this configure when the generation of checkpoint barrier is not frequent.
804        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
805        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
806        // 32MB
807        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
808        const DEFAULT_LEVEL_MULTIPLIER: u64 = 10;
809        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
810        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 128;
811        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
812        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
813        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
814        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
815        const DEFAULT_EMERGENCY_PICKER: bool = true;
816        const DEFAULT_MAX_LEVEL: u32 = 6;
817        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
818        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
819        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 256;
820        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
821        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
822        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 5000;
823        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
824        const DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION: bool = true;
825        const DEFAULT_VNODE_ALIGNED_LEVEL_SIZE_THRESHOLD: Option<u64> = None;
826        pub const DEFAULT_MAX_KV_COUNT_FOR_XOR16: u64 = 256 * 1024;
827
828        use crate::catalog::hummock::CompactionFilterFlag;
829
830        pub fn max_bytes_for_level_base() -> u64 {
831            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
832        }
833
834        pub fn max_bytes_for_level_multiplier() -> u64 {
835            DEFAULT_LEVEL_MULTIPLIER
836        }
837
838        pub fn max_compaction_bytes() -> u64 {
839            DEFAULT_MAX_COMPACTION_BYTES
840        }
841
842        pub fn sub_level_max_compaction_bytes() -> u64 {
843            DEFAULT_MIN_COMPACTION_BYTES
844        }
845
846        pub fn level0_tier_compact_file_number() -> u64 {
847            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
848        }
849
850        pub fn target_file_size_base() -> u64 {
851            DEFAULT_TARGET_FILE_SIZE_BASE
852        }
853
854        pub fn compaction_filter_mask() -> u32 {
855            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
856        }
857
858        pub fn max_sub_compaction() -> u32 {
859            DEFAULT_MAX_SUB_COMPACTION
860        }
861
862        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
863            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
864        }
865
866        pub fn level0_sub_level_compact_level_count() -> u32 {
867            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
868        }
869
870        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
871            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
872        }
873
874        pub fn max_space_reclaim_bytes() -> u64 {
875            DEFAULT_MAX_SPACE_RECLAIM_BYTES
876        }
877
878        pub fn level0_max_compact_file_number() -> u64 {
879            DEFAULT_MAX_COMPACTION_FILE_COUNT
880        }
881
882        pub fn tombstone_reclaim_ratio() -> u32 {
883            DEFAULT_TOMBSTONE_RATIO_PERCENT
884        }
885
886        pub fn enable_emergency_picker() -> bool {
887            DEFAULT_EMERGENCY_PICKER
888        }
889
890        pub fn max_level() -> u32 {
891            DEFAULT_MAX_LEVEL
892        }
893
894        pub fn max_l0_compact_level_count() -> u32 {
895            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
896        }
897
898        pub fn sst_allowed_trivial_move_min_size() -> u64 {
899            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
900        }
901
902        pub fn disable_auto_group_scheduling() -> bool {
903            false
904        }
905
906        pub fn max_overlapping_level_size() -> u64 {
907            256 * MB
908        }
909
910        pub fn sst_allowed_trivial_move_max_count() -> u32 {
911            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
912        }
913
914        pub fn emergency_level0_sst_file_count() -> u32 {
915            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
916        }
917
918        pub fn emergency_level0_sub_level_partition() -> u32 {
919            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
920        }
921
922        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
923            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
924        }
925
926        pub fn level0_stop_write_threshold_max_size() -> u64 {
927            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
928        }
929
930        pub fn enable_optimize_l0_interval_selection() -> bool {
931            DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION
932        }
933
934        pub fn vnode_aligned_level_size_threshold() -> Option<u64> {
935            DEFAULT_VNODE_ALIGNED_LEVEL_SIZE_THRESHOLD
936        }
937
938        pub fn max_kv_count_for_xor16() -> Option<u64> {
939            Some(DEFAULT_MAX_KV_COUNT_FOR_XOR16)
940        }
941    }
942}
risingwave_common/config/meta.rs

risingwave_common/config/
meta.rs
