risingwave_common/config/
meta.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_common_proc_macro::serde_prefix_all;
16
17use super::*;
18
19#[derive(Copy, Clone, Debug, Default, ValueEnum, Serialize, Deserialize)]
20pub enum MetaBackend {
21    #[default]
22    Mem,
23    Sql, // any database url
24    Sqlite,
25    Postgres,
26    Mysql,
27}
28
29/// Compression algorithm for hummock version checkpoint serialization.
30#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "lowercase")]
32#[repr(i32)]
33pub enum CheckpointCompression {
34    /// No compression.
35    ///
36    /// NOTE: The numeric values are aligned with protobuf `CheckpointCompressionAlgorithm`.
37    None = 0,
38    /// Zstd compression (default, good balance between ratio and speed).
39    #[default]
40    Zstd = 1,
41    /// Lz4 compression (faster but lower ratio).
42    Lz4 = 2,
43}
44
45#[cfg(test)]
46mod tests {
47    use risingwave_pb::hummock::CheckpointCompressionAlgorithm;
48
49    use super::CheckpointCompression;
50
51    #[test]
52    fn checkpoint_compression_numeric_values_align_with_pb() {
53        assert_eq!(
54            CheckpointCompression::None as i32,
55            CheckpointCompressionAlgorithm::CheckpointCompressionUnspecified as i32
56        );
57        assert_eq!(
58            CheckpointCompression::Zstd as i32,
59            CheckpointCompressionAlgorithm::CheckpointCompressionZstd as i32
60        );
61        assert_eq!(
62            CheckpointCompression::Lz4 as i32,
63            CheckpointCompressionAlgorithm::CheckpointCompressionLz4 as i32
64        );
65    }
66}
67
68#[derive(Copy, Clone, Debug, Default)]
69pub enum DefaultParallelism {
70    #[default]
71    Full,
72    Default(NonZeroUsize),
73}
74
75impl Serialize for DefaultParallelism {
76    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
77    where
78        S: Serializer,
79    {
80        #[derive(Debug, Serialize, Deserialize)]
81        #[serde(untagged)]
82        enum Parallelism {
83            Str(String),
84            Int(usize),
85        }
86        match self {
87            DefaultParallelism::Full => Parallelism::Str("Full".to_owned()).serialize(serializer),
88            DefaultParallelism::Default(val) => {
89                Parallelism::Int(val.get() as _).serialize(serializer)
90            }
91        }
92    }
93}
94
95impl<'de> Deserialize<'de> for DefaultParallelism {
96    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
97    where
98        D: serde::Deserializer<'de>,
99    {
100        #[derive(Debug, Deserialize)]
101        #[serde(untagged)]
102        enum Parallelism {
103            Str(String),
104            Int(usize),
105        }
106        let p = Parallelism::deserialize(deserializer)?;
107        match p {
108            Parallelism::Str(s) => {
109                if s.trim().eq_ignore_ascii_case("full") {
110                    Ok(DefaultParallelism::Full)
111                } else {
112                    Err(serde::de::Error::custom(format!(
113                        "invalid default parallelism: {}",
114                        s
115                    )))
116                }
117            }
118            Parallelism::Int(i) => Ok(DefaultParallelism::Default(
119                // Note: we won't check whether this exceeds the maximum parallelism (i.e., vnode count)
120                // here because it requires extra context. The check will be done when scheduling jobs.
121                NonZeroUsize::new(i).ok_or_else(|| {
122                    serde::de::Error::custom("default parallelism should not be 0")
123                })?,
124            )),
125        }
126    }
127}
128
129/// The section `[meta]` in `risingwave.toml`.
130#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
131#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
132pub struct MetaConfig {
133    /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they
134    /// are dangling.
135    #[serde(default = "default::meta::min_sst_retention_time_sec")]
136    pub min_sst_retention_time_sec: u64,
137
138    /// Interval of automatic hummock full GC.
139    #[serde(default = "default::meta::full_gc_interval_sec")]
140    pub full_gc_interval_sec: u64,
141
142    /// Max number of object per full GC job can fetch.
143    #[serde(default = "default::meta::full_gc_object_limit")]
144    pub full_gc_object_limit: u64,
145
146    /// Duration in seconds to retain garbage collection history data.
147    #[serde(default = "default::meta::gc_history_retention_time_sec")]
148    pub gc_history_retention_time_sec: u64,
149
150    /// Max number of inflight time travel query.
151    #[serde(default = "default::meta::max_inflight_time_travel_query")]
152    pub max_inflight_time_travel_query: u64,
153
154    /// Schedule `Dynamic` compaction for all compaction groups with this interval.
155    /// Groups in cooldown (recently found to have no compaction work) are skipped.
156    #[serde(default = "default::meta::periodic_compaction_interval_sec")]
157    pub periodic_compaction_interval_sec: u64,
158
159    /// Interval of invoking a vacuum job, to remove stale metadata from meta store and objects
160    /// from object store.
161    #[serde(default = "default::meta::vacuum_interval_sec")]
162    pub vacuum_interval_sec: u64,
163
164    /// The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of
165    /// meta node.
166    #[serde(default = "default::meta::vacuum_spin_interval_ms")]
167    pub vacuum_spin_interval_ms: u64,
168
169    /// Interval of invoking iceberg garbage collection, to expire old snapshots.
170    #[serde(default = "default::meta::iceberg_gc_interval_sec")]
171    pub iceberg_gc_interval_sec: u64,
172
173    /// Interval of hummock version checkpoint.
174    #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")]
175    pub hummock_version_checkpoint_interval_sec: u64,
176
177    /// Compression algorithm for hummock version checkpoint.
178    #[serde(default)]
179    pub checkpoint_compression_algorithm: CheckpointCompression,
180
181    /// Chunk size in bytes for reading large checkpoints.
182    /// Large checkpoints are read in parallel chunks to avoid single-request timeout issues.
183    /// Default: 128MB
184    #[serde(default = "default::meta::checkpoint_read_chunk_size")]
185    pub checkpoint_read_chunk_size: usize,
186
187    /// Maximum number of concurrent chunk reads when reading large checkpoints.
188    /// Higher values may improve read throughput but increase memory usage.
189    /// Memory usage = `checkpoint_read_chunk_size` * `checkpoint_read_max_in_flight_chunks`
190    /// Default: 4
191    #[serde(default = "default::meta::checkpoint_read_max_in_flight_chunks")]
192    pub checkpoint_read_max_in_flight_chunks: usize,
193
194    /// If enabled, `SSTable` object file and version delta will be retained.
195    ///
196    /// `SSTable` object file need to be deleted via full GC.
197    ///
198    /// version delta need to be manually deleted.
199    #[serde(default = "default::meta::enable_hummock_data_archive")]
200    pub enable_hummock_data_archive: bool,
201
202    /// The interval at which a Hummock version snapshot is taken for time travel.
203    ///
204    /// Larger value indicates less storage overhead but worse query performance.
205    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
206    pub hummock_time_travel_snapshot_interval: u64,
207
208    /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
209    /// attempt is rejected.
210    #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
211    pub min_delta_log_num_for_hummock_version_checkpoint: u64,
212
213    /// Maximum allowed heartbeat interval in seconds.
214    #[serde(default = "default::meta::max_heartbeat_interval_sec")]
215    pub max_heartbeat_interval_secs: u32,
216
217    /// Whether to enable fail-on-recovery. Should only be used in e2e tests.
218    #[serde(default)]
219    pub disable_recovery: bool,
220
221    /// Whether meta should request pausing all data sources on the next bootstrap.
222    /// This allows us to pause the cluster on next bootstrap in an offline way.
223    /// It's important for standalone or single node deployments.
224    /// In those cases, meta node, frontend and compute may all be co-located.
225    /// If the compute node enters an inconsistent state, and continuously crashloops,
226    /// we may not be able to connect to the cluster to run `alter system set pause_on_next_bootstrap = true;`.
227    /// By providing it in the static config, we can have an offline way to trigger the pause on bootstrap.
228    #[serde(default = "default::meta::pause_on_next_bootstrap_offline")]
229    pub pause_on_next_bootstrap_offline: bool,
230
231    /// Whether to disable adaptive-scaling feature.
232    #[serde(default)]
233    pub disable_automatic_parallelism_control: bool,
234
235    /// The number of streaming jobs per scaling operation.
236    #[serde(default = "default::meta::parallelism_control_batch_size")]
237    pub parallelism_control_batch_size: usize,
238
239    /// The period of parallelism control trigger.
240    #[serde(default = "default::meta::parallelism_control_trigger_period_sec")]
241    pub parallelism_control_trigger_period_sec: u64,
242
243    /// The first delay of parallelism control.
244    #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")]
245    pub parallelism_control_trigger_first_delay_sec: u64,
246
247    #[serde(default = "default::meta::meta_leader_lease_secs")]
248    pub meta_leader_lease_secs: u64,
249
250    /// After specified seconds of idle (no mview or flush), the process will be exited.
251    /// It is mainly useful for playgrounds.
252    #[serde(default)]
253    pub dangerous_max_idle_secs: Option<u64>,
254
255    /// The default global parallelism for all streaming jobs, if user doesn't specify the
256    /// parallelism, this value will be used. `FULL` means use all available parallelism units,
257    /// otherwise it's a number.
258    #[serde(default = "default::meta::default_parallelism")]
259    pub default_parallelism: DefaultParallelism,
260
261    /// Whether to enable deterministic compaction scheduling, which
262    /// will disable all auto scheduling of compaction tasks.
263    /// Should only be used in e2e tests.
264    #[serde(default)]
265    pub enable_compaction_deterministic: bool,
266
267    /// Enable sanity check when SSTs are committed.
268    #[serde(default)]
269    pub enable_committed_sst_sanity_check: bool,
270
271    #[serde(default = "default::meta::node_num_monitor_interval_sec")]
272    pub node_num_monitor_interval_sec: u64,
273
274    #[serde(default = "default::meta::backend")]
275    pub backend: MetaBackend,
276
277    /// Schedule `space_reclaim` compaction for all compaction groups with this interval.
278    #[serde(default = "default::meta::periodic_space_reclaim_compaction_interval_sec")]
279    pub periodic_space_reclaim_compaction_interval_sec: u64,
280
281    /// Schedule `ttl_reclaim` compaction for all compaction groups with this interval.
282    #[serde(default = "default::meta::periodic_ttl_reclaim_compaction_interval_sec")]
283    pub periodic_ttl_reclaim_compaction_interval_sec: u64,
284
285    #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
286    pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
287
288    #[serde(default = "default::meta::move_table_size_limit")]
289    #[deprecated]
290    pub move_table_size_limit: u64,
291
292    #[serde(default = "default::meta::split_group_size_limit")]
293    #[deprecated]
294    pub split_group_size_limit: u64,
295
296    #[serde(default = "default::meta::cut_table_size_limit")]
297    #[deprecated]
298    pub cut_table_size_limit: u64,
299
300    /// Whether to protect dropping a table with incoming sink.
301    #[serde(default = "default::meta::protect_drop_table_with_incoming_sink")]
302    pub protect_drop_table_with_incoming_sink: bool,
303
304    #[serde(default, flatten)]
305    #[config_doc(omitted)]
306    pub unrecognized: Unrecognized<Self>,
307
308    /// Whether config object storage bucket lifecycle to purge stale data.
309    #[serde(default)]
310    pub do_not_config_object_storage_lifecycle: bool,
311
312    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
313    /// Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table.
314    #[serde(default = "default::meta::partition_vnode_count")]
315    pub partition_vnode_count: u32,
316
317    /// The threshold of write throughput to trigger a group split.
318    #[serde(
319        default = "default::meta::table_high_write_throughput_threshold",
320        alias = "table_write_throughput_threshold"
321    )]
322    pub table_high_write_throughput_threshold: u64,
323
324    #[serde(
325        default = "default::meta::table_low_write_throughput_threshold",
326        alias = "min_table_split_write_throughput"
327    )]
328    /// The threshold of write throughput to trigger a group merge.
329    pub table_low_write_throughput_threshold: u64,
330
331    // If the compaction task does not report heartbeat beyond the
332    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
333    #[serde(default = "default::meta::compaction_task_max_heartbeat_interval_secs")]
334    pub compaction_task_max_heartbeat_interval_secs: u64,
335
336    // If the compaction task does not change in progress beyond the
337    // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
338    #[serde(default = "default::meta::compaction_task_max_progress_interval_secs")]
339    pub compaction_task_max_progress_interval_secs: u64,
340
341    #[serde(default)]
342    #[config_doc(nested)]
343    pub compaction_config: CompactionConfig,
344
345    /// Count of partitions of tables in default group and materialized view group.
346    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
347    /// Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
348    /// Set it zero to disable this feature.
349    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
350    pub hybrid_partition_vnode_count: u32,
351
352    #[serde(default = "default::meta::event_log_enabled")]
353    pub event_log_enabled: bool,
354    /// Keeps the latest N events per channel.
355    #[serde(default = "default::meta::event_log_channel_max_size")]
356    pub event_log_channel_max_size: u32,
357
358    #[serde(default)]
359    #[config_doc(nested)]
360    pub developer: MetaDeveloperConfig,
361
362    /// Whether compactor should rewrite row to remove dropped column.
363    #[serde(default = "default::meta::enable_dropped_column_reclaim")]
364    pub enable_dropped_column_reclaim: bool,
365
366    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
367    #[serde(default = "default::meta::split_group_size_ratio")]
368    pub split_group_size_ratio: f64,
369
370    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
371    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
372    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
373    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
374    pub table_stat_high_write_throughput_ratio_for_split: f64,
375
376    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
377    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
378    pub table_stat_low_write_throughput_ratio_for_merge: f64,
379
380    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
381    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
382    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
383    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
384    /// The window seconds of table throughput statistic history for split compaction group.
385    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
386    pub table_stat_throuput_window_seconds_for_split: usize,
387
388    /// The window seconds of table throughput statistic history for merge compaction group.
389    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
390    pub table_stat_throuput_window_seconds_for_merge: usize,
391
392    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
393    /// Set it max value of 64-bit number to disable this feature.
394    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
395    pub compact_task_table_size_partition_threshold_low: u64,
396
397    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
398    /// Set it max value of 64-bit number to disable this feature.
399    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
400    pub compact_task_table_size_partition_threshold_high: u64,
401
402    /// The interval of the periodic scheduling compaction group split job.
403    #[serde(
404        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
405        alias = "periodic_split_compact_group_interval_sec"
406    )]
407    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
408
409    /// The interval of the periodic scheduling compaction group merge job.
410    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
411    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
412
413    /// The threshold of each dimension of the compaction group after merging. When the dimension * `compaction_group_merge_dimension_threshold` >= limit, the merging job will be rejected.
414    #[serde(default = "default::meta::compaction_group_merge_dimension_threshold")]
415    pub compaction_group_merge_dimension_threshold: f64,
416
417    /// The interval that the CDC table splits initialization should yield to avoid overloading upstream system.
418    #[serde(default = "default::meta::cdc_table_split_init_sleep_interval_splits")]
419    pub cdc_table_split_init_sleep_interval_splits: u64,
420
421    /// The duration that the CDC table splits initialization should yield to avoid overloading upstream system.
422    #[serde(default = "default::meta::cdc_table_split_init_sleep_duration_millis")]
423    pub cdc_table_split_init_sleep_duration_millis: u64,
424
425    /// The batch size that the CDC table splits initialization should use when persisting to meta store.
426    #[serde(default = "default::meta::cdc_table_split_init_insert_batch_size")]
427    pub cdc_table_split_init_insert_batch_size: u64,
428
429    /// Whether to automatically migrate legacy table fragments when meta starts.
430    #[serde(default = "default::meta::enable_legacy_table_migration")]
431    pub enable_legacy_table_migration: bool,
432
433    #[serde(default)]
434    #[config_doc(nested)]
435    pub meta_store_config: MetaStoreConfig,
436}
437
438/// Note: only applies to meta store backends other than `SQLite`.
439#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
440#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
441pub struct MetaStoreConfig {
442    /// Maximum number of connections for the meta store connection pool.
443    #[serde(default = "default::meta_store_config::max_connections")]
444    pub max_connections: u32,
445    /// Minimum number of connections for the meta store connection pool.
446    #[serde(default = "default::meta_store_config::min_connections")]
447    pub min_connections: u32,
448    /// Connection timeout in seconds for a meta store connection.
449    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
450    pub connection_timeout_sec: u64,
451    /// Idle timeout in seconds for a meta store connection.
452    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
453    pub idle_timeout_sec: u64,
454    /// Acquire timeout in seconds for a meta store connection.
455    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
456    pub acquire_timeout_sec: u64,
457}
458
459/// The subsections `[meta.developer]`.
460///
461/// It is put at [`MetaConfig::developer`].
462#[serde_prefix_all("meta_", mode = "alias")]
463#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
464#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
465pub struct MetaDeveloperConfig {
466    /// The number of traces to be cached in-memory by the tracing collector
467    /// embedded in the meta node.
468    #[serde(default = "default::developer::meta_cached_traces_num")]
469    pub cached_traces_num: u32,
470
471    /// The maximum memory usage in bytes for the tracing collector embedded
472    /// in the meta node.
473    #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")]
474    pub cached_traces_memory_limit_bytes: usize,
475
476    /// Compaction picker config
477    #[serde(default = "default::developer::enable_trivial_move")]
478    pub enable_trivial_move: bool,
479    #[serde(default = "default::developer::enable_check_task_level_overlap")]
480    pub enable_check_task_level_overlap: bool,
481    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
482    pub max_trivial_move_task_count_per_loop: usize,
483
484    #[serde(default = "default::developer::max_get_task_probe_times")]
485    pub max_get_task_probe_times: usize,
486
487    /// Max number of actor allowed per parallelism (default = 100).
488    /// CREATE MV/Table will be noticed when the number of actors exceeds this limit.
489    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_soft_limit")]
490    pub actor_cnt_per_worker_parallelism_soft_limit: usize,
491
492    /// Max number of actor allowed per parallelism (default = 400).
493    /// CREATE MV/Table will be rejected when the number of actors exceeds this limit.
494    #[serde(default = "default::developer::actor_cnt_per_worker_parallelism_hard_limit")]
495    pub actor_cnt_per_worker_parallelism_hard_limit: usize,
496
497    /// Max number of SSTs fetched from meta store per SELECT, during time travel Hummock version replay.
498    #[serde(default = "default::developer::hummock_time_travel_sst_info_fetch_batch_size")]
499    pub hummock_time_travel_sst_info_fetch_batch_size: usize,
500
501    /// Max number of SSTs inserted into meta store per INSERT, during time travel metadata writing.
502    #[serde(default = "default::developer::hummock_time_travel_sst_info_insert_batch_size")]
503    pub hummock_time_travel_sst_info_insert_batch_size: usize,
504
505    #[serde(default = "default::developer::time_travel_vacuum_interval_sec")]
506    pub time_travel_vacuum_interval_sec: u64,
507
508    #[serde(default = "default::developer::time_travel_vacuum_max_version_count")]
509    pub time_travel_vacuum_max_version_count: Option<u32>,
510
511    /// Max number of epoch-to-version inserted into meta store per INSERT, during time travel metadata writing.
512    #[serde(default = "default::developer::hummock_time_travel_epoch_version_insert_batch_size")]
513    pub hummock_time_travel_epoch_version_insert_batch_size: usize,
514
515    #[serde(default = "default::developer::hummock_gc_history_insert_batch_size")]
516    pub hummock_gc_history_insert_batch_size: usize,
517
518    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_batch_size")]
519    pub hummock_time_travel_filter_out_objects_batch_size: usize,
520
521    #[serde(default = "default::developer::hummock_time_travel_filter_out_objects_v1")]
522    pub hummock_time_travel_filter_out_objects_v1: bool,
523
524    #[serde(
525        default = "default::developer::hummock_time_travel_filter_out_objects_list_version_batch_size"
526    )]
527    pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
528
529    #[serde(
530        default = "default::developer::hummock_time_travel_filter_out_objects_list_delta_batch_size"
531    )]
532    pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
533
534    #[serde(default)]
535    pub compute_client_config: RpcClientConfig,
536
537    #[serde(default)]
538    pub stream_client_config: RpcClientConfig,
539
540    #[serde(default)]
541    pub frontend_client_config: RpcClientConfig,
542}
543
544#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
545#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
546pub struct CompactionConfig {
547    #[serde(default = "default::compaction_config::max_bytes_for_level_base")]
548    pub max_bytes_for_level_base: u64,
549    #[serde(default = "default::compaction_config::max_bytes_for_level_multiplier")]
550    pub max_bytes_for_level_multiplier: u64,
551    #[serde(default = "default::compaction_config::max_compaction_bytes")]
552    pub max_compaction_bytes: u64,
553    #[serde(default = "default::compaction_config::sub_level_max_compaction_bytes")]
554    pub sub_level_max_compaction_bytes: u64,
555    #[serde(default = "default::compaction_config::level0_tier_compact_file_number")]
556    pub level0_tier_compact_file_number: u64,
557    #[serde(default = "default::compaction_config::target_file_size_base")]
558    pub target_file_size_base: u64,
559    #[serde(default = "default::compaction_config::compaction_filter_mask")]
560    pub compaction_filter_mask: u32,
561    #[serde(default = "default::compaction_config::max_sub_compaction")]
562    pub max_sub_compaction: u32,
563    #[serde(default = "default::compaction_config::level0_stop_write_threshold_sub_level_number")]
564    pub level0_stop_write_threshold_sub_level_number: u64,
565    #[serde(default = "default::compaction_config::level0_sub_level_compact_level_count")]
566    pub level0_sub_level_compact_level_count: u32,
567    #[serde(
568        default = "default::compaction_config::level0_overlapping_sub_level_compact_level_count"
569    )]
570    pub level0_overlapping_sub_level_compact_level_count: u32,
571    #[serde(default = "default::compaction_config::max_space_reclaim_bytes")]
572    pub max_space_reclaim_bytes: u64,
573    #[serde(default = "default::compaction_config::level0_max_compact_file_number")]
574    pub level0_max_compact_file_number: u64,
575    #[serde(default = "default::compaction_config::tombstone_reclaim_ratio")]
576    pub tombstone_reclaim_ratio: u32,
577    #[serde(default = "default::compaction_config::enable_emergency_picker")]
578    pub enable_emergency_picker: bool,
579    #[serde(default = "default::compaction_config::max_level")]
580    pub max_level: u32,
581    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_min_size")]
582    pub sst_allowed_trivial_move_min_size: u64,
583    #[serde(default = "default::compaction_config::sst_allowed_trivial_move_max_count")]
584    pub sst_allowed_trivial_move_max_count: u32,
585    #[serde(default = "default::compaction_config::max_l0_compact_level_count")]
586    pub max_l0_compact_level_count: u32,
587    #[serde(default = "default::compaction_config::disable_auto_group_scheduling")]
588    pub disable_auto_group_scheduling: bool,
589    #[serde(default = "default::compaction_config::max_overlapping_level_size")]
590    pub max_overlapping_level_size: u64,
591    #[serde(default = "default::compaction_config::emergency_level0_sst_file_count")]
592    pub emergency_level0_sst_file_count: u32,
593    #[serde(default = "default::compaction_config::emergency_level0_sub_level_partition")]
594    pub emergency_level0_sub_level_partition: u32,
595    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_sst_count")]
596    pub level0_stop_write_threshold_max_sst_count: u32,
597    #[serde(default = "default::compaction_config::level0_stop_write_threshold_max_size")]
598    pub level0_stop_write_threshold_max_size: u64,
599    #[serde(default = "default::compaction_config::enable_optimize_l0_interval_selection")]
600    pub enable_optimize_l0_interval_selection: bool,
601    #[serde(default = "default::compaction_config::max_kv_count_for_xor16")]
602    pub max_kv_count_for_xor16: Option<u64>,
603    #[serde(default = "default::compaction_config::max_vnode_key_range_bytes")]
604    pub max_vnode_key_range_bytes: Option<u64>,
605}
606
607pub mod default {
608    pub use crate::config::default::developer;
609
610    pub mod meta {
611        use crate::config::{DefaultParallelism, MetaBackend};
612
613        pub fn min_sst_retention_time_sec() -> u64 {
614            3600 * 6
615        }
616
617        pub fn gc_history_retention_time_sec() -> u64 {
618            3600 * 6
619        }
620
621        pub fn full_gc_interval_sec() -> u64 {
622            3600
623        }
624
625        pub fn full_gc_object_limit() -> u64 {
626            100_000
627        }
628
629        pub fn max_inflight_time_travel_query() -> u64 {
630            1000
631        }
632
633        pub fn periodic_compaction_interval_sec() -> u64 {
634            300
635        }
636
637        pub fn vacuum_interval_sec() -> u64 {
638            30
639        }
640
641        pub fn vacuum_spin_interval_ms() -> u64 {
642            100
643        }
644
645        pub fn iceberg_gc_interval_sec() -> u64 {
646            3600
647        }
648
649        pub fn hummock_version_checkpoint_interval_sec() -> u64 {
650            30
651        }
652
653        pub fn checkpoint_read_chunk_size() -> usize {
654            128 * 1024 * 1024 // 128MB
655        }
656
657        pub fn checkpoint_read_max_in_flight_chunks() -> usize {
658            4
659        }
660
661        pub fn enable_hummock_data_archive() -> bool {
662            false
663        }
664
665        pub fn hummock_time_travel_snapshot_interval() -> u64 {
666            100
667        }
668
669        pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
670            10
671        }
672
673        pub fn max_heartbeat_interval_sec() -> u32 {
674            60
675        }
676
677        pub fn meta_leader_lease_secs() -> u64 {
678            30
679        }
680
681        pub fn default_parallelism() -> DefaultParallelism {
682            DefaultParallelism::Full
683        }
684
685        pub fn pause_on_next_bootstrap_offline() -> bool {
686            false
687        }
688
689        pub fn node_num_monitor_interval_sec() -> u64 {
690            10
691        }
692
693        pub fn backend() -> MetaBackend {
694            MetaBackend::Mem
695        }
696
697        pub fn periodic_space_reclaim_compaction_interval_sec() -> u64 {
698            3600 // 60min
699        }
700
701        pub fn periodic_ttl_reclaim_compaction_interval_sec() -> u64 {
702            1800 // 30mi
703        }
704
705        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
706            10 // 10s
707        }
708
709        pub fn periodic_tombstone_reclaim_compaction_interval_sec() -> u64 {
710            600
711        }
712
713        // limit the size of state table to trigger split by high throughput
714        pub fn move_table_size_limit() -> u64 {
715            10 * 1024 * 1024 * 1024 // 10GB
716        }
717
718        // limit the size of group to trigger split by group_size and avoid too many small groups
719        pub fn split_group_size_limit() -> u64 {
720            64 * 1024 * 1024 * 1024 // 64GB
721        }
722
723        pub fn protect_drop_table_with_incoming_sink() -> bool {
724            false
725        }
726
727        pub fn partition_vnode_count() -> u32 {
728            16
729        }
730
731        pub fn table_high_write_throughput_threshold() -> u64 {
732            16 * 1024 * 1024 // 16MB
733        }
734
735        pub fn table_low_write_throughput_threshold() -> u64 {
736            4 * 1024 * 1024 // 4MB
737        }
738
739        pub fn compaction_task_max_heartbeat_interval_secs() -> u64 {
740            30 // 30s
741        }
742
743        pub fn compaction_task_max_progress_interval_secs() -> u64 {
744            60 * 10 // 10min
745        }
746
747        pub fn cut_table_size_limit() -> u64 {
748            1024 * 1024 * 1024 // 1GB
749        }
750
751        pub fn hybrid_partition_vnode_count() -> u32 {
752            4
753        }
754
755        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
756            128 * 1024 * 1024 // 128MB
757        }
758
759        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
760            512 * 1024 * 1024 // 512MB
761        }
762
763        pub fn event_log_enabled() -> bool {
764            true
765        }
766
767        pub fn event_log_channel_max_size() -> u32 {
768            10
769        }
770
771        pub fn parallelism_control_batch_size() -> usize {
772            10
773        }
774
775        pub fn parallelism_control_trigger_period_sec() -> u64 {
776            10
777        }
778
779        pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
780            30
781        }
782
783        pub fn enable_dropped_column_reclaim() -> bool {
784            false
785        }
786
787        pub fn split_group_size_ratio() -> f64 {
788            0.9
789        }
790
791        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
792            0.5
793        }
794
795        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
796            0.7
797        }
798
799        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
800            60
801        }
802
803        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
804            240
805        }
806
807        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
808            60 * 10 // 10min
809        }
810
811        pub fn compaction_group_merge_dimension_threshold() -> f64 {
812            1.2
813        }
814
815        pub fn cdc_table_split_init_sleep_interval_splits() -> u64 {
816            1000
817        }
818
819        pub fn cdc_table_split_init_sleep_duration_millis() -> u64 {
820            500
821        }
822
823        pub fn cdc_table_split_init_insert_batch_size() -> u64 {
824            100
825        }
826
827        pub fn enable_legacy_table_migration() -> bool {
828            true
829        }
830    }
831
832    pub mod meta_store_config {
833        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
834        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
835        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
836        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
837        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
838
839        pub fn max_connections() -> u32 {
840            DEFAULT_MAX_CONNECTIONS
841        }
842
843        pub fn min_connections() -> u32 {
844            DEFAULT_MIN_CONNECTIONS
845        }
846
847        pub fn connection_timeout_sec() -> u64 {
848            DEFAULT_CONNECTION_TIMEOUT_SEC
849        }
850
851        pub fn idle_timeout_sec() -> u64 {
852            DEFAULT_IDLE_TIMEOUT_SEC
853        }
854
855        pub fn acquire_timeout_sec() -> u64 {
856            DEFAULT_ACQUIRE_TIMEOUT_SEC
857        }
858    }
859
860    pub mod compaction_config {
861        const MB: u64 = 1024 * 1024;
862        const GB: u64 = 1024 * 1024 * 1024;
863        const DEFAULT_MAX_COMPACTION_BYTES: u64 = 2 * GB; // 2GB
864        const DEFAULT_MIN_COMPACTION_BYTES: u64 = 128 * MB; // 128MB
865        const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * MB; // 512MB
866
867        // decrease this configure when the generation of checkpoint barrier is not frequent.
868        const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
869        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * MB;
870        // 32MB
871        const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
872        const DEFAULT_LEVEL_MULTIPLIER: u64 = 10;
873        const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * MB; // 512MB;
874        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 128;
875        const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100;
876        const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
877        const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
878        const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
879        const DEFAULT_EMERGENCY_PICKER: bool = true;
880        const DEFAULT_MAX_LEVEL: u32 = 6;
881        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
882        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE: u64 = 4 * MB;
883        const DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT: u32 = 256;
884        const DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT: u32 = 2000; // > 50G / 32M = 1600
885        const DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION: u32 = 256;
886        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT: u32 = 5000;
887        const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE: u64 = 300 * 1024 * MB; // 300GB
888        const DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION: bool = true;
889        pub const DEFAULT_MAX_KV_COUNT_FOR_XOR16: u64 = 256 * 1024;
890        const DEFAULT_MAX_VNODE_KEY_RANGE_BYTES: Option<u64> = None;
891
892        use crate::catalog::hummock::CompactionFilterFlag;
893
894        pub fn max_bytes_for_level_base() -> u64 {
895            DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
896        }
897
898        pub fn max_bytes_for_level_multiplier() -> u64 {
899            DEFAULT_LEVEL_MULTIPLIER
900        }
901
902        pub fn max_compaction_bytes() -> u64 {
903            DEFAULT_MAX_COMPACTION_BYTES
904        }
905
906        pub fn sub_level_max_compaction_bytes() -> u64 {
907            DEFAULT_MIN_COMPACTION_BYTES
908        }
909
910        pub fn level0_tier_compact_file_number() -> u64 {
911            DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
912        }
913
914        pub fn target_file_size_base() -> u64 {
915            DEFAULT_TARGET_FILE_SIZE_BASE
916        }
917
918        pub fn compaction_filter_mask() -> u32 {
919            (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
920        }
921
922        pub fn max_sub_compaction() -> u32 {
923            DEFAULT_MAX_SUB_COMPACTION
924        }
925
926        pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
927            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
928        }
929
930        pub fn level0_sub_level_compact_level_count() -> u32 {
931            DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
932        }
933
934        pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
935            DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
936        }
937
938        pub fn max_space_reclaim_bytes() -> u64 {
939            DEFAULT_MAX_SPACE_RECLAIM_BYTES
940        }
941
942        pub fn level0_max_compact_file_number() -> u64 {
943            DEFAULT_MAX_COMPACTION_FILE_COUNT
944        }
945
946        pub fn tombstone_reclaim_ratio() -> u32 {
947            DEFAULT_TOMBSTONE_RATIO_PERCENT
948        }
949
950        pub fn enable_emergency_picker() -> bool {
951            DEFAULT_EMERGENCY_PICKER
952        }
953
954        pub fn max_level() -> u32 {
955            DEFAULT_MAX_LEVEL
956        }
957
958        pub fn max_l0_compact_level_count() -> u32 {
959            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
960        }
961
962        pub fn sst_allowed_trivial_move_min_size() -> u64 {
963            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MIN_SIZE
964        }
965
966        pub fn disable_auto_group_scheduling() -> bool {
967            false
968        }
969
970        pub fn max_overlapping_level_size() -> u64 {
971            256 * MB
972        }
973
974        pub fn sst_allowed_trivial_move_max_count() -> u32 {
975            DEFAULT_SST_ALLOWED_TRIVIAL_MOVE_MAX_COUNT
976        }
977
978        pub fn emergency_level0_sst_file_count() -> u32 {
979            DEFAULT_EMERGENCY_LEVEL0_SST_FILE_COUNT
980        }
981
982        pub fn emergency_level0_sub_level_partition() -> u32 {
983            DEFAULT_EMERGENCY_LEVEL0_SUB_LEVEL_PARTITION
984        }
985
986        pub fn level0_stop_write_threshold_max_sst_count() -> u32 {
987            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SST_COUNT
988        }
989
990        pub fn level0_stop_write_threshold_max_size() -> u64 {
991            DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_MAX_SIZE
992        }
993
994        pub fn enable_optimize_l0_interval_selection() -> bool {
995            DEFAULT_ENABLE_OPTIMIZE_L0_INTERVAL_SELECTION
996        }
997
998        pub fn max_kv_count_for_xor16() -> Option<u64> {
999            Some(DEFAULT_MAX_KV_COUNT_FOR_XOR16)
1000        }
1001
1002        pub fn max_vnode_key_range_bytes() -> Option<u64> {
1003            DEFAULT_MAX_VNODE_KEY_RANGE_BYTES
1004        }
1005    }
1006}
risingwave_common/config/meta.rs

risingwave_common/config/
meta.rs
