risingwave_common/config/
meta.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common_proc_macro::serde_prefix_all;
16use serde::de::Error as _;
17
18use super::*;
19
20#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
21pub enum MetaBackend {
22    #[default]
23    Mem,
24    Sql, // any database url
25    Sqlite,
26    Postgres,
27    Mysql,
28}
29
30/// Compression algorithm for hummock version checkpoint serialization.
31#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(rename_all = "lowercase")]
33#[repr(i32)]
34pub enum CheckpointCompression {
35    /// No compression.
36    ///
37    /// NOTE: The numeric values are aligned with protobuf `CheckpointCompressionAlgorithm`.
38    None = 0,
39    /// Zstd compression (default, good balance between ratio and speed).
40    #[default]
41    Zstd = 1,
42    /// Lz4 compression (faster but lower ratio).
43    Lz4 = 2,
44}
45
46#[cfg(test)]
47mod tests {
48    use risingwave_pb::hummock::CheckpointCompressionAlgorithm;
49
50    use super::CheckpointCompression;
51
52    #[test]
53    fn checkpoint_compression_numeric_values_align_with_pb() {
54        assert_eq!(
55            CheckpointCompression::None as i32,
56            CheckpointCompressionAlgorithm::CheckpointCompressionUnspecified as i32
57        );
58        assert_eq!(
59            CheckpointCompression::Zstd as i32,
60            CheckpointCompressionAlgorithm::CheckpointCompressionZstd as i32
61        );
62        assert_eq!(
63            CheckpointCompression::Lz4 as i32,
64            CheckpointCompressionAlgorithm::CheckpointCompressionLz4 as i32
65        );
66    }
67}
68
69#[derive(Copy, Clone, Debug, Default)]
70pub enum DefaultParallelism {
71    #[default]
72    Full,
73    Default(NonZeroUsize),
74}
75
76impl Serialize for DefaultParallelism {
77    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
78    where
79        S: Serializer,
80    {
81        #[derive(Debug, Serialize, Deserialize)]
82        #[serde(untagged)]
83        enum Parallelism {
84            Str(String),
85            Int(usize),
86        }
87        match self {
88            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
89            DefaultParallelism::Default(val) => {
90                Parallelism::Int(val.get() as _).serialize(serializer)
91            }
92        }
93    }
94}
95
96impl<'de> Deserialize<'de> for DefaultParallelism {
97    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
98    where
99        D: serde::Deserializer<'de>,
100    {
101        #[derive(Debug, Deserialize)]
102        #[serde(untagged)]
103        enum Parallelism {
104            Str(String),
105            Int(usize),
106        }
107        let p = Parallelism::deserialize(deserializer)?;
108        match p {
109            Parallelism::Str(s) => {
110                if s.trim().eq_ignore_ascii_case("full") {
111                    Ok(DefaultParallelism::Full)
112                } else {
113                    Err(serde::de::Error::custom(format!(
114                        "invalid default parallelism: {}",
115                        s
116                    )))
117                }
118            }
119            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
120                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
121                // here because it requires extra context. The check will be done when scheduling jobs.
122                NonZeroUsize::new(i).ok_or_else(|| {
123                    serde::de::Error::custom("default parallelism should not be 0")
124                })?,
125            )),
126        }
127    }
128}
129
130/// The section `[meta]` in `risingwave.toml`.
131#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
132#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
133pub struct MetaConfig {
134    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
135    /// are dangling.
136    #[serde(default = "default::meta::min_sst_retention_time_sec")]
137    pub min_sst_retention_time_sec: u64,
138
139    /// Interval of automatic hummock full GC.
140    #[serde(default = "default::meta::full_gc_interval_sec")]
141    pub full_gc_interval_sec: u64,
142
143    /// Max number of object per full GC job can fetch.
144    #[serde(default = "default::meta::full_gc_object_limit")]
145    pub full_gc_object_limit: u64,
146
147    /// Duration in seconds to retain garbage collection history data.
148    #[serde(default = "default::meta::gc_history_retention_time_sec")]
149    pub gc_history_retention_time_sec: u64,
150
151    /// Max number of inflight time travel query.
152    #[serde(default = "default::meta::max_inflight_time_travel_query")]
153    pub max_inflight_time_travel_query: u64,
154
155    /// Schedule `Dynamic` compaction for all compaction groups with this interval.
156    /// Groups in cooldown (recently found to have no compaction work) are skipped.
157    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
158    pub periodic_compaction_interval_sec: u64,
159
160    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
161    /// from object store.
162    #[serde(default = "default::meta::vacuum_interval_sec")]
163    pub vacuum_interval_sec: u64,
164
165    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
166    /// meta node.
167    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
168    pub vacuum_spin_interval_ms: u64,
169
170    /// Interval of invoking iceberg garbage collection, to expire old snapshots.
171    #[serde(default = "default::meta::iceberg_gc_interval_sec")]
172    pub iceberg_gc_interval_sec: u64,
173
174    /// Maximum time to wait for an iceberg compaction task report before the lease expires.
175    #[serde(default = "default::meta::iceberg_compaction_report_timeout_sec")]
176    pub iceberg_compaction_report_timeout_sec: u64,
177
178    /// Maximum time to reuse cached iceberg compaction schedule config before refreshing it from
179    /// meta catalog.
180    #[serde(default = "default::meta::iceberg_compaction_config_refresh_interval_sec")]
181    pub iceberg_compaction_config_refresh_interval_sec: u64,
182
183    /// Interval of hummock version checkpoint.
184    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
185    pub hummock_version_checkpoint_interval_sec: u64,
186
187    /// Compression algorithm for hummock version checkpoint.
188    #[serde(default)]
189    pub checkpoint_compression_algorithm: CheckpointCompression,
190
191    /// Chunk size in bytes for reading large checkpoints.
192    /// Large checkpoints are read in parallel chunks to avoid single-request timeout issues.
193    /// Default: 128MB
194    #[serde(default = "default::meta::checkpoint_read_chunk_size")]
195    pub checkpoint_read_chunk_size: usize,
196
197    /// Maximum number of concurrent chunk reads when reading large checkpoints.
198    /// Higher values may improve read throughput but increase memory usage.
199    /// Memory usage = `checkpoint_read_chunk_size` * `checkpoint_read_max_in_flight_chunks`
200    /// Default: 4
201    #[serde(default = "default::meta::checkpoint_read_max_in_flight_chunks")]
202    pub checkpoint_read_max_in_flight_chunks: usize,
203
204    /// If enabled, `SSTable` object file and version delta will be retained.
205    ///
206    /// `SSTable` object file need to be deleted via full GC.
207    ///
208    /// version delta need to be manually deleted.
209    #[serde(default = "default::meta::enable_hummock_data_archive")]
210    pub enable_hummock_data_archive: bool,
211
212    /// The interval at which a Hummock version snapshot is taken for time travel.
213    ///
214    /// Larger value indicates less storage overhead but worse query performance.
215    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
216    pub hummock_time_travel_snapshot_interval: u64,
217
218    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
219    /// attempt is rejected.
220    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
221    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
222
223    /// Maximum allowed heartbeat interval in seconds.
224    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
225    pub max_heartbeat_interval_secs: u32,
226
227    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
228    #[serde(default)]
229    pub disable_recovery: bool,
230
231    /// Whether meta should request pausing all data sources on the next bootstrap.
232    /// This allows us to pause the cluster on next bootstrap in an offline way.
233    /// It's important for standalone or single node deployments.
234    /// In those cases, meta node, frontend and compute may all be co-located.
235    /// If the compute node enters an inconsistent state, and continuously crashloops,
236    /// we may not be able to connect to the cluster to run `alter system set pause_on_next_bootstrap = true;`.
237    /// By providing it in the static config, we can have an offline way to trigger the pause on bootstrap.
238    #[serde(default = "default::meta::pause_on_next_bootstrap_offline")]
239    pub pause_on_next_bootstrap_offline: bool,
240
241    /// Whether to disable adaptive-scaling feature.
242    #[serde(default)]
243    pub disable_automatic_parallelism_control: bool,
244
245    /// The number of streaming jobs per scaling operation.
246    #[serde(default = "default::meta::parallelism_control_batch_size")]
247    pub parallelism_control_batch_size: usize,
248
249    /// The period of parallelism control trigger.
250    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
251    pub parallelism_control_trigger_period_sec: u64,
252
253    /// The first delay of parallelism control.
254    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
255    pub parallelism_control_trigger_first_delay_sec: u64,
256
257    #[serde(default = "default::meta::meta_leader_lease_secs")]
258    pub meta_leader_lease_secs: u64,
259
260    /// After specified seconds of idle (no mview or flush), the process will be exited.
261    /// It is mainly useful for playgrounds.
262    #[serde(default)]
263    pub dangerous_max_idle_secs: Option<u64>,
264
265    /// The default global parallelism for all streaming jobs, if user doesn't specify the
266    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
267    /// otherwise it's a number.
268    #[serde(default = "default::meta::default_parallelism")]
269    pub default_parallelism: DefaultParallelism,
270
271    /// Whether to enable deterministic compaction scheduling, which
272    /// will disable all auto scheduling of compaction tasks.
273    /// Should only be used in e2e tests.
274    #[serde(default)]
275    pub enable_compaction_deterministic: bool,
276
277    /// Enable sanity check when SSTs are committed.
278    #[serde(default)]
279    pub enable_committed_sst_sanity_check: bool,
280
281    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
282    pub node_num_monitor_interval_sec: u64,
283
284    #[serde(default = "default::meta::backend")]
285    pub backend: MetaBackend,
286
287    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
288    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
289    pub periodic_space_reclaim_compaction_interval_sec: u64,
290
291    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
292    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
293    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
294
295    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
296    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
297
298    #[serde(default = "default::meta::move_table_size_limit")]
299    #[deprecated]
300    pub move_table_size_limit: u64,
301
302    #[serde(default = "default::meta::split_group_size_limit")]
303    #[deprecated]
304    pub split_group_size_limit: u64,
305
306    #[serde(default = "default::meta::cut_table_size_limit")]
307    #[deprecated]
308    pub cut_table_size_limit: u64,
309
310    /// Whether to protect dropping a table with incoming sink.
311    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
312    pub protect_drop_table_with_incoming_sink: bool,
313
314    #[serde(default, flatten)]
315    #[config_doc(omitted)]
316    pub unrecognized: Unrecognized<Self>,
317
318    /// Whether config object storage bucket lifecycle to purge stale data.
319    #[serde(default)]
320    pub do_not_config_object_storage_lifecycle: bool,
321
322    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
323    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
324    #[serde(default = "default::meta::partition_vnode_count")]
325    pub partition_vnode_count: u32,
326
327    /// The threshold of write throughput to trigger a group split.
328    #[serde(
329        default = "default::meta::table_high_write_throughput_threshold",
330        alias = "table_write_throughput_threshold"
331    )]
332    pub table_high_write_throughput_threshold: u64,
333
334    #[serde(
335        default = "default::meta::table_low_write_throughput_threshold",
336        alias = "min_table_split_write_throughput"
337    )]
338    /// The threshold of write throughput to trigger a group merge.
339    pub table_low_write_throughput_threshold: u64,
340
341    // If the compaction task does not report heartbeat beyond the
342    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
343    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
344    pub compaction_task_max_heartbeat_interval_secs: u64,
345
346    // If the compaction task does not change in progress beyond the
347    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
348    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
349    pub compaction_task_max_progress_interval_secs: u64,
350
351    /// The number of compaction task ids to prefetch from the meta store in one batch.
352    #[serde(default = "default::meta::compaction_task_id_refill_capacity")]
353    pub compaction_task_id_refill_capacity: u32,
354
355    #[serde(default)]
356    #[config_doc(nested)]
357    pub compaction_config: CompactionConfig,
358
359    /// Count of partitions of tables in default group and materialized view group.
360    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
361    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
362    /// Set it zero to disable this feature.
363    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
364    pub hybrid_partition_vnode_count: u32,
365
366    #[serde(default = "default::meta::event_log_enabled")]
367    pub event_log_enabled: bool,
368    /// Keeps the latest N events per channel.
369    #[serde(default = "default::meta::event_log_channel_max_size")]
370    pub event_log_channel_max_size: u32,
371
372    #[serde(default)]
373    #[config_doc(nested)]
374    pub developer: MetaDeveloperConfig,
375
376    /// Whether compactor should rewrite row to remove dropped column.
377    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
378    pub enable_dropped_column_reclaim: bool,
379
380    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
381    #[serde(default = "default::meta::split_group_size_ratio")]
382    pub split_group_size_ratio: f64,
383
384    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
385    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
386    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
387    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
388    pub table_stat_high_write_throughput_ratio_for_split: f64,
389
390    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
391    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
392    pub table_stat_low_write_throughput_ratio_for_merge: f64,
393
394    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
395    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
396    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
397    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
398    /// The window seconds of table throughput statistic history for split compaction group.
399    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
400    pub table_stat_throuput_window_seconds_for_split: usize,
401
402    /// The window seconds of table throughput statistic history for merge compaction group.
403    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
404    pub table_stat_throuput_window_seconds_for_merge: usize,
405
406    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
407    /// Set it max value of 64-bit number to disable this feature.
408    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
409    pub compact_task_table_size_partition_threshold_low: u64,
410
411    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
412    /// Set it max value of 64-bit number to disable this feature.
413    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
414    pub compact_task_table_size_partition_threshold_high: u64,
415
416    /// The interval of the regular periodic compaction group split job.
417    /// This does not disable merge-triggered normalize splits when
418    /// `enable_compaction_group_normalize` is enabled.
419    #[serde(
420        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
421        alias = "periodic_split_compact_group_interval_sec"
422    )]
423    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
424
425    /// Whether to normalize overlapping compaction groups before the regular merge scheduling.
426    #[serde(default = "default::meta::enable_compaction_group_normalize")]
427    pub enable_compaction_group_normalize: bool,
428
429    /// The maximum number of normalize splits in one scheduler round. Must be greater than 0.
430    #[serde(
431        default = "default::meta::max_normalize_splits_per_round",
432        deserialize_with = "deserialize_max_normalize_splits_per_round"
433    )]
434    pub max_normalize_splits_per_round: u64,
435
436    /// The interval of the periodic scheduling compaction group merge job.
437    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
438    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
439
440    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
441    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
442    pub compaction_group_merge_dimension_threshold: f64,
443
444    /// The interval that the CDC table splits initialization should yield to avoid overloading upstream system.
445    #[serde(default = "default::meta::cdc_table_split_init_sleep_interval_splits")]
446    pub cdc_table_split_init_sleep_interval_splits: u64,
447
448    /// The duration that the CDC table splits initialization should yield to avoid overloading upstream system.
449    #[serde(default = "default::meta::cdc_table_split_init_sleep_duration_millis")]
450    pub cdc_table_split_init_sleep_duration_millis: u64,
451
452    /// The batch size that the CDC table splits initialization should use when persisting to meta store.
453    #[serde(default = "default::meta::cdc_table_split_init_insert_batch_size")]
454    pub cdc_table_split_init_insert_batch_size: u64,
455
456    /// Whether to automatically migrate legacy table fragments when meta starts.
457    #[serde(default = "default::meta::enable_legacy_table_migration")]
458    pub enable_legacy_table_migration: bool,
459
460    #[serde(default)]
461    #[config_doc(nested)]
462    pub meta_store_config: MetaStoreConfig,
463}
464
465/// Note: only applies to meta store backends other than `SQLite`.
466#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
467#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
468pub struct MetaStoreConfig {
469    /// Maximum number of connections for the meta store connection pool.
470    #[serde(default = "default::meta_store_config::max_connections")]
471    pub max_connections: u32,
472    /// Minimum number of connections for the meta store connection pool.
473    #[serde(default = "default::meta_store_config::min_connections")]
474    pub min_connections: u32,
475    /// Connection timeout in seconds for a meta store connection.
476    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
477    pub connection_timeout_sec: u64,
478    /// Idle timeout in seconds for a meta store connection.
479    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
480    pub idle_timeout_sec: u64,
481    /// Acquire timeout in seconds for a meta store connection.
482    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
483    pub acquire_timeout_sec: u64,
484}
485
486fn deserialize_max_normalize_splits_per_round<'de, D>(deserializer: D) -> Result<u64, D::Error>
487where
488    D: serde::Deserializer<'de>,
489{
490    let value = u64::deserialize(deserializer)?;
491    if value == 0 {
492        return Err(D::Error::custom(
493            "meta.max_normalize_splits_per_round must be greater than 0",
494        ));
495    }
496    Ok(value)
497}
498
499/// The subsections `[meta.developer]`.
500///
501/// It is put at [`MetaConfig::developer`].
502#[serde_prefix_all("meta_", mode = "alias")]
503#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
504#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
505pub struct MetaDeveloperConfig {
506    /// The number of traces to be cached in-memory by the tracing collector
507    /// embedded in the meta node.
508    #[serde(default = "default::developer::meta_cached_traces_num")]
509    pub cached_traces_num: u32,
510
511    /// The maximum memory usage in bytes for the tracing collector embedded
512    /// in the meta node.
513    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
514    pub cached_traces_memory_limit_bytes: usize,
515
516    /// Compaction picker config
517    #[serde(default = "default::developer::enable_trivial_move")]
518    pub enable_trivial_move: bool,
519    #[serde(default = "default::developer::enable_check_task_level_overlap")]
520    pub enable_check_task_level_overlap: bool,
521    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
522    pub max_trivial_move_task_count_per_loop: usize,
523
524    #[serde(default = "default::developer::max_get_task_probe_times")]
525    pub max_get_task_probe_times: usize,
526
527    /// Max number of actor allowed per parallelism (default = 100).
528    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
529    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
530    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
531
532    /// Max number of actor allowed per parallelism (default = 400).
533    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
534    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
535    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
536
537    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
538    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
539    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
540
541    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
542    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
543    pub hummock_time_travel_sst_info_insert_batch_size: usize,
544
545    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
546    pub time_travel_vacuum_interval_sec: u64,
547
548    #[serde(default = "default::developer::time_travel_vacuum_max_version_count")]
549    pub time_travel_vacuum_max_version_count: Option<u32>,
550
551    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
552    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
553    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
554
555    /// Max number of version deltas fetched from meta store per SELECT, during time travel metadata vacuum.
556    #[serde(default = "default::developer::hummock_time_travel_delta_fetch_batch_size")]
557    pub hummock_time_travel_delta_fetch_batch_size: usize,
558
559    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
560    pub hummock_gc_history_insert_batch_size: usize,
561
562    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
563    pub hummock_time_travel_filter_out_objects_batch_size: usize,
564
565    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
566    pub hummock_time_travel_filter_out_objects_v1: bool,
567
568    #[serde(
569        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
570    )]
571    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
572
573    #[serde(
574        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
575    )]
576    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
577
578    #[serde(default)]
579    pub compute_client_config: RpcClientConfig,
580
581    #[serde(default)]
582    pub stream_client_config: RpcClientConfig,
583
584    #[serde(default)]
585    pub frontend_client_config: RpcClientConfig,
586
587    #[serde(default = "default::developer::table_change_log_insert_batch_size")]
588    pub table_change_log_insert_batch_size: u64,
589
590    #[serde(default = "default::developer::table_change_log_delete_batch_size")]
591    pub table_change_log_delete_batch_size: u64,
592}
593
594#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
595#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
596pub struct CompactionConfig {
597    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
598    pub max_bytes_for_level_base: u64,
599    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
600    pub max_bytes_for_level_multiplier: u64,
601    #[serde(default = "default::compaction_config::max_compaction_bytes")]
602    pub max_compaction_bytes: u64,
603    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
604    pub sub_level_max_compaction_bytes: u64,
605    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
606    pub level0_tier_compact_file_number: u64,
607    #[serde(default = "default::compaction_config::target_file_size_base")]
608    pub target_file_size_base: u64,
609    #[serde(default = "default::compaction_config::compaction_filter_mask")]
610    pub compaction_filter_mask: u32,
611    #[serde(default = "default::compaction_config::max_sub_compaction")]
612    pub max_sub_compaction: u32,
613    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
614    pub level0_stop_write_threshold_sub_level_number: u64,
615    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
616    pub level0_sub_level_compact_level_count: u32,
617    #[serde(
618        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
619    )]
620    pub level0_overlapping_sub_level_compact_level_count: u32,
621    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
622    pub max_space_reclaim_bytes: u64,
623    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
624    pub level0_max_compact_file_number: u64,
625    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
626    pub tombstone_reclaim_ratio: u32,
627    #[serde(default = "default::compaction_config::enable_emergency_picker")]
628    pub enable_emergency_picker: bool,
629    #[serde(default = "default::compaction_config::max_level")]
630    pub max_level: u32,
631    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
632    pub sst_allowed_trivial_move_min_size: u64,
633    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
634    pub sst_allowed_trivial_move_max_count: u32,
635    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
636    pub max_l0_compact_level_count: u32,
637    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
638    pub disable_auto_group_scheduling: bool,
639    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
640    pub max_overlapping_level_size: u64,
641    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
642    pub emergency_level0_sst_file_count: u32,
643    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
644    pub emergency_level0_sub_level_partition: u32,
645    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
646    pub level0_stop_write_threshold_max_sst_count: u32,
647    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
648    pub level0_stop_write_threshold_max_size: u64,
649    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
650    pub enable_optimize_l0_interval_selection: bool,
651    #[serde(default = "default::compaction_config::max_kv_count_for_xor16")]
652    pub max_kv_count_for_xor16: Option<u64>,
653    #[serde(default = "default::compaction_config::max_vnode_key_range_bytes")]
654    pub max_vnode_key_range_bytes: Option<u64>,
655}
656
657pub mod default {
658    pub use crate::config::default::developer;
659
660    pub mod meta {
661        use crate::config::{DefaultParallelism, MetaBackend};
662
663        pub fn min_sst_retention_time_sec() -> u64 {
664            3600 * 6
665        }
666
667        pub fn gc_history_retention_time_sec() -> u64 {
668            3600 * 6
669        }
670
671        pub fn full_gc_interval_sec() -> u64 {
672            3600
673        }
674
675        pub fn full_gc_object_limit() -> u64 {
676            100_000
677        }
678
679        pub fn max_inflight_time_travel_query() -> u64 {
680            1000
681        }
682
683        pub fn periodic_compaction_interval_sec() -> u64 {
684            300
685        }
686
687        pub fn vacuum_interval_sec() -> u64 {
688            30
689        }
690
691        pub fn vacuum_spin_interval_ms() -> u64 {
692            100
693        }
694
695        pub fn iceberg_gc_interval_sec() -> u64 {
696            3600
697        }
698
699        pub fn iceberg_compaction_report_timeout_sec() -> u64 {
700            30 * 60
701        }
702
703        pub fn iceberg_compaction_config_refresh_interval_sec() -> u64 {
704            60
705        }
706
707        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
708            30
709        }
710
711        pub fn checkpoint_read_chunk_size() -> usize {
712            128 * 1024 * 1024 // 128MB
713        }
714
715        pub fn checkpoint_read_max_in_flight_chunks() -> usize {
716            4
717        }
718
719        pub fn enable_hummock_data_archive() -> bool {
720            false
721        }
722
723        pub fn hummock_time_travel_snapshot_interval() -> u64 {
724            100
725        }
726
727        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
728            10
729        }
730
731        pub fn max_heartbeat_interval_sec() -> u32 {
732            60
733        }
734
735        pub fn meta_leader_lease_secs() -> u64 {
736            30
737        }
738
739        pub fn default_parallelism() -> DefaultParallelism {
740            DefaultParallelism::Full
741        }
742
743        pub fn pause_on_next_bootstrap_offline() -> bool {
744            false
745        }
746
747        pub fn node_num_monitor_interval_sec() -> u64 {
748            10
749        }
750
751        pub fn backend() -> MetaBackend {
752            MetaBackend::Mem
753        }
754
755        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
756            3600 // 60min
757        }
758
759        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
760            1800 // 30mi
761        }
762
763        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
764            10 // 10s
765        }
766
767        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
768            600
769        }
770
771        // limit the size of state table to trigger split by high throughput
772        pub fn move_table_size_limit() -> u64 {
773            10 * 1024 * 1024 * 1024 // 10GB
774        }
775
776        // limit the size of group to trigger split by group_size and avoid too many small groups
777        pub fn split_group_size_limit() -> u64 {
778            64 * 1024 * 1024 * 1024 // 64GB
779        }
780
781        pub fn protect_drop_table_with_incoming_sink() -> bool {
782            false
783        }
784
785        pub fn partition_vnode_count() -> u32 {
786            16
787        }
788
789        pub fn table_high_write_throughput_threshold() -> u64 {
790            16 * 1024 * 1024 // 16MB
791        }
792
793        pub fn table_low_write_throughput_threshold() -> u64 {
794            4 * 1024 * 1024 // 4MB
795        }
796
797        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
798            30 // 30s
799        }
800
801        pub fn compaction_task_max_progress_interval_secs() -> u64 {
802            60 * 10 // 10min
803        }
804
805        pub fn compaction_task_id_refill_capacity() -> u32 {
806            64
807        }
808
809        pub fn cut_table_size_limit() -> u64 {
810            1024 * 1024 * 1024 // 1GB
811        }
812
813        pub fn hybrid_partition_vnode_count() -> u32 {
814            4
815        }
816
817        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
818            128 * 1024 * 1024 // 128MB
819        }
820
821        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
822            512 * 1024 * 1024 // 512MB
823        }
824
825        pub fn event_log_enabled() -> bool {
826            true
827        }
828
829        pub fn event_log_channel_max_size() -> u32 {
830            10
831        }
832
833        pub fn parallelism_control_batch_size() -> usize {
834            10
835        }
836
837        pub fn parallelism_control_trigger_period_sec() -> u64 {
838            10
839        }
840
841        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
842            30
843        }
844
845        pub fn enable_dropped_column_reclaim() -> bool {
846            false
847        }
848
849        pub fn split_group_size_ratio() -> f64 {
850            0.9
851        }
852
853        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
854            0.5
855        }
856
857        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
858            0.7
859        }
860
861        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
862            60
863        }
864
865        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
866            240
867        }
868
869        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
870            60 * 10 // 10min
871        }
872
873        pub fn enable_compaction_group_normalize() -> bool {
874            true
875        }
876
877        pub fn max_normalize_splits_per_round() -> u64 {
878            4
879        }
880
881        pub fn compaction_group_merge_dimension_threshold() -> f64 {
882            1.2
883        }
884
885        pub fn cdc_table_split_init_sleep_interval_splits() -> u64 {
886            1000
887        }
888
889        pub fn cdc_table_split_init_sleep_duration_millis() -> u64 {
890            500
891        }
892
893        pub fn cdc_table_split_init_insert_batch_size() -> u64 {
894            100
895        }
896
897        pub fn enable_legacy_table_migration() -> bool {
898            true
899        }
900    }
901
902    pub mod meta_store_config {
903        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
904        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
905        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
906        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
907        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
908
909        pub fn max_connections() -> u32 {
910            DEFAULT_MAX_CONNECTIONS
911        }
912
913        pub fn min_connections() -> u32 {
914            DEFAULT_MIN_CONNECTIONS
915        }
916
917        pub fn connection_timeout_sec() -> u64 {
918            DEFAULT_CONNECTION_TIMEOUT_SEC
919        }
920
921        pub fn idle_timeout_sec() -> u64 {
922            DEFAULT_IDLE_TIMEOUT_SEC
923        }
924
925        pub fn acquire_timeout_sec() -> u64 {
926            DEFAULT_ACQUIRE_TIMEOUT_SEC
927        }
928    }
929
930    pub mod compaction_config {
931        const MB: u64 = 1024 * 1024;
932        const GB: u64 = 1024 * 1024 * 1024;
933        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
934        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
935        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
936
937        // decrease this configure when the generation of checkpoint barrier is not frequent.
938        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
939        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
940        // 32MB
941        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
942        const DEFAULT_LEVEL_MULTIPLIER: u64 = 10;
943        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
944        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 128;
945        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
946        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
947        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
948        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
949        const DEFAULT_EMERGENCY_PICKER: bool = true;
950        const DEFAULT_MAX_LEVEL: u32 = 6;
951        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
952        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
953        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 256;
954        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
955        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
956        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 5000;
957        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
958        const DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION: bool = true;
959        pub const DEFAULT_MAX_KV_COUNT_FOR_XOR16: u64 = 256 * 1024;
960        const DEFAULT_MAX_VNODE_KEY_RANGE_BYTES: Option<u64> = None;
961
962        use crate::catalog::hummock::CompactionFilterFlag;
963
964        pub fn max_bytes_for_level_base() -> u64 {
965            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
966        }
967
968        pub fn max_bytes_for_level_multiplier() -> u64 {
969            DEFAULT_LEVEL_MULTIPLIER
970        }
971
972        pub fn max_compaction_bytes() -> u64 {
973            DEFAULT_MAX_COMPACTION_BYTES
974        }
975
976        pub fn sub_level_max_compaction_bytes() -> u64 {
977            DEFAULT_MIN_COMPACTION_BYTES
978        }
979
980        pub fn level0_tier_compact_file_number() -> u64 {
981            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
982        }
983
984        pub fn target_file_size_base() -> u64 {
985            DEFAULT_TARGET_FILE_SIZE_BASE
986        }
987
988        pub fn compaction_filter_mask() -> u32 {
989            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
990        }
991
992        pub fn max_sub_compaction() -> u32 {
993            DEFAULT_MAX_SUB_COMPACTION
994        }
995
996        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
997            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
998        }
999
1000        pub fn level0_sub_level_compact_level_count() -> u32 {
1001            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
1002        }
1003
1004        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
1005            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
1006        }
1007
1008        pub fn max_space_reclaim_bytes() -> u64 {
1009            DEFAULT_MAX_SPACE_RECLAIM_BYTES
1010        }
1011
1012        pub fn level0_max_compact_file_number() -> u64 {
1013            DEFAULT_MAX_COMPACTION_FILE_COUNT
1014        }
1015
1016        pub fn tombstone_reclaim_ratio() -> u32 {
1017            DEFAULT_TOMBSTONE_RATIO_PERCENT
1018        }
1019
1020        pub fn enable_emergency_picker() -> bool {
1021            DEFAULT_EMERGENCY_PICKER
1022        }
1023
1024        pub fn max_level() -> u32 {
1025            DEFAULT_MAX_LEVEL
1026        }
1027
1028        pub fn max_l0_compact_level_count() -> u32 {
1029            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
1030        }
1031
1032        pub fn sst_allowed_trivial_move_min_size() -> u64 {
1033            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
1034        }
1035
1036        pub fn disable_auto_group_scheduling() -> bool {
1037            false
1038        }
1039
1040        pub fn max_overlapping_level_size() -> u64 {
1041            256 * MB
1042        }
1043
1044        pub fn sst_allowed_trivial_move_max_count() -> u32 {
1045            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
1046        }
1047
1048        pub fn emergency_level0_sst_file_count() -> u32 {
1049            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
1050        }
1051
1052        pub fn emergency_level0_sub_level_partition() -> u32 {
1053            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
1054        }
1055
1056        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
1057            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
1058        }
1059
1060        pub fn level0_stop_write_threshold_max_size() -> u64 {
1061            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
1062        }
1063
1064        pub fn enable_optimize_l0_interval_selection() -> bool {
1065            DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION
1066        }
1067
1068        pub fn max_kv_count_for_xor16() -> Option<u64> {
1069            Some(DEFAULT_MAX_KV_COUNT_FOR_XOR16)
1070        }
1071
1072        pub fn max_vnode_key_range_bytes() -> Option<u64> {
1073            DEFAULT_MAX_VNODE_KEY_RANGE_BYTES
1074        }
1075    }
1076}
risingwave_common/config/meta.rs

risingwave_common/config/
meta.rs
