Skip to main content

risingwave_common/config/
mod.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! This pub module defines the structure of the configuration file `risingwave.toml`.
16//!
17//! [`RwConfig`] corresponds to the whole config file and each other config struct corresponds to a
18//! section in `risingwave.toml`.
19
20pub mod batch;
21pub use batch::BatchConfig;
22pub mod frontend;
23pub use frontend::FrontendConfig;
24pub mod hba;
25pub use hba::{AddressPattern, AuthMethod, ConnectionType, HbaConfig, HbaEntry};
26pub mod meta;
27pub use meta::{
28    CheckpointCompression, CompactionConfig, DefaultParallelism, MetaBackend, MetaConfig,
29    MetaStoreConfig,
30};
31pub mod streaming;
32pub use streaming::{AsyncStackTraceOption, StreamingConfig};
33pub mod server;
34pub use server::{HeapProfilingConfig, ServerConfig};
35
36pub use crate::session_config::SessionInitConfig;
37pub mod udf;
38pub use udf::UdfConfig;
39pub mod storage;
40pub use storage::{
41    CacheEvictionConfig, EvictionConfig, ObjectStoreConfig, StorageConfig, StorageMemoryConfig,
42    extract_storage_memory_config,
43};
44pub mod merge;
45pub mod mutate;
46pub mod none_as_empty_string;
47pub mod role;
48pub mod system;
49pub mod utils;
50use std::collections::BTreeMap;
51use std::fs;
52use std::num::NonZeroUsize;
53
54use anyhow::Context;
55use clap::ValueEnum;
56use educe::Educe;
57pub use merge::*;
58use risingwave_common_proc_macro::ConfigDoc;
59pub use risingwave_common_proc_macro::OverrideConfig;
60use risingwave_pb::meta::SystemParams;
61pub use role::*;
62use serde::{Deserialize, Serialize, Serializer};
63use serde_default::DefaultFromSerde;
64use serde_json::Value;
65pub use system::SystemConfig;
66pub use utils::*;
67
68use crate::for_all_params;
69
70/// Use the maximum value for HTTP/2 connection window size to avoid deadlock among multiplexed
71/// streams on the same connection.
72pub const MAX_CONNECTION_WINDOW_SIZE: u32 = (1 << 31) - 1;
73/// Use a large value for HTTP/2 stream window size to improve the performance of remote exchange,
74/// as we don't rely on this for back-pressure.
75pub const STREAM_WINDOW_SIZE: u32 = 32 * 1024 * 1024; // 32 MB
76
77/// [`RwConfig`] corresponds to the whole config file `risingwave.toml`. Each field corresponds to a
78/// section.
79#[derive(Educe, Clone, Serialize, Deserialize, Default, ConfigDoc)]
80#[educe(Debug)]
81pub struct RwConfig {
82    #[serde(default)]
83    #[config_doc(nested)]
84    pub server: ServerConfig,
85
86    #[serde(default)]
87    #[config_doc(nested)]
88    pub meta: MetaConfig,
89
90    #[serde(default)]
91    #[config_doc(nested)]
92    pub batch: BatchConfig,
93
94    #[serde(default)]
95    #[config_doc(nested)]
96    pub frontend: FrontendConfig,
97
98    #[serde(default)]
99    #[config_doc(nested)]
100    pub streaming: StreamingConfig,
101
102    #[serde(default)]
103    #[config_doc(nested)]
104    pub storage: StorageConfig,
105
106    #[serde(default)]
107    #[educe(Debug(ignore))]
108    #[config_doc(nested)]
109    pub system: SystemConfig,
110
111    #[serde(default)]
112    #[config_doc(nested)]
113    pub udf: UdfConfig,
114
115    #[serde(default)]
116    #[config_doc(nested)]
117    pub session_init: SessionInitConfig,
118
119    #[serde(flatten)]
120    #[config_doc(omitted)]
121    pub unrecognized: Unrecognized<Self>,
122}
123
124/// `[meta.developer.meta_compute_client_config]`
125/// `[meta.developer.meta_stream_client_config]`
126/// `[meta.developer.meta_frontend_client_config]`
127/// `[batch.developer.batch_compute_client_config]`
128/// `[batch.developer.batch_frontend_client_config]`
129/// `[streaming.developer.stream_compute_client_config]`
130#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
131#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
132pub struct RpcClientConfig {
133    #[serde(default = "default::developer::rpc_client_connect_timeout_secs")]
134    pub connect_timeout_secs: u64,
135    /// Maximum concurrency when setting up an RPC client pool.
136    /// Set to 0 to keep the previous unlimited behavior.
137    #[serde(default = "default::developer::rpc_client_pool_setup_concurrency")]
138    pub pool_setup_concurrency: usize,
139}
140
141pub use risingwave_common_metrics::MetricLevel;
142
143impl RwConfig {
144    pub const fn default_connection_pool_size(&self) -> u16 {
145        self.server.connection_pool_size
146    }
147
148    /// Returns [`streaming::StreamingDeveloperConfig::exchange_connection_pool_size`] if set,
149    /// otherwise [`ServerConfig::connection_pool_size`].
150    pub fn streaming_exchange_connection_pool_size(&self) -> u16 {
151        self.streaming
152            .developer
153            .exchange_connection_pool_size
154            .unwrap_or_else(|| self.default_connection_pool_size())
155    }
156
157    /// Returns [`batch::BatchDeveloperConfig::exchange_connection_pool_size`] if set,
158    /// otherwise [`ServerConfig::connection_pool_size`].
159    pub fn batch_exchange_connection_pool_size(&self) -> u16 {
160        self.batch
161            .developer
162            .exchange_connection_pool_size
163            .unwrap_or_else(|| self.default_connection_pool_size())
164    }
165}
166
167pub mod default {
168
169    pub mod developer {
170        use crate::config::streaming::CacheRefillPolicy;
171
172        pub fn meta_cached_traces_num() -> u32 {
173            256
174        }
175
176        pub fn meta_cached_traces_memory_limit_bytes() -> usize {
177            1 << 27 // 128 MiB
178        }
179
180        pub fn batch_output_channel_size() -> usize {
181            64
182        }
183
184        pub fn batch_receiver_channel_size() -> usize {
185            1000
186        }
187
188        pub fn batch_root_stage_channel_size() -> usize {
189            100
190        }
191
192        pub fn batch_chunk_size() -> usize {
193            1024
194        }
195
196        pub fn batch_local_execute_buffer_size() -> usize {
197            64
198        }
199
200        /// Default to unset to be compatible with the behavior before this config is introduced,
201        /// that is, follow the value of `server.connection_pool_size`.
202        pub fn batch_exchange_connection_pool_size() -> Option<u16> {
203            None
204        }
205
206        pub fn stream_enable_executor_row_count() -> bool {
207            false
208        }
209
210        pub fn connector_message_buffer_size() -> usize {
211            16
212        }
213
214        pub fn unsafe_stream_extreme_cache_size() -> usize {
215            10
216        }
217
218        pub fn stream_topn_cache_min_capacity() -> usize {
219            10
220        }
221
222        pub fn stream_chunk_size() -> usize {
223            256
224        }
225
226        pub fn stream_exchange_initial_permits() -> usize {
227            2048
228        }
229
230        pub fn stream_exchange_batched_permits() -> usize {
231            256
232        }
233
234        pub fn stream_exchange_concurrent_barriers() -> usize {
235            1
236        }
237
238        pub fn stream_exchange_concurrent_dispatchers() -> usize {
239            0
240        }
241
242        pub fn stream_project_expr_concurrency() -> usize {
243            1
244        }
245
246        pub fn stream_project_expr_inflight_request_concurrency() -> usize {
247            0
248        }
249
250        pub fn stream_dml_channel_initial_permits() -> usize {
251            32768
252        }
253
254        pub fn stream_max_barrier_batch_size() -> u32 {
255            1024
256        }
257
258        pub fn stream_hash_agg_max_dirty_groups_heap_size() -> usize {
259            64 << 20 // 64MB
260        }
261
262        pub fn enable_trivial_move() -> bool {
263            true
264        }
265
266        pub fn enable_check_task_level_overlap() -> bool {
267            false
268        }
269
270        pub fn max_trivial_move_task_count_per_loop() -> usize {
271            256
272        }
273
274        pub fn max_get_task_probe_times() -> usize {
275            5
276        }
277
278        pub fn actor_cnt_per_worker_parallelism_soft_limit() -> usize {
279            100
280        }
281
282        pub fn actor_cnt_per_worker_parallelism_hard_limit() -> usize {
283            400
284        }
285
286        pub fn hummock_time_travel_sst_info_fetch_batch_size() -> usize {
287            10_000
288        }
289
290        pub fn hummock_time_travel_sst_info_insert_batch_size() -> usize {
291            100
292        }
293
294        pub fn time_travel_vacuum_interval_sec() -> u64 {
295            30
296        }
297
298        pub fn time_travel_vacuum_max_version_count() -> Option<u32> {
299            Some(10000)
300        }
301
302        pub fn hummock_time_travel_epoch_version_insert_batch_size() -> usize {
303            1000
304        }
305
306        pub fn hummock_time_travel_delta_fetch_batch_size() -> usize {
307            100
308        }
309
310        pub fn hummock_gc_history_insert_batch_size() -> usize {
311            1000
312        }
313
314        pub fn hummock_time_travel_filter_out_objects_batch_size() -> usize {
315            1000
316        }
317
318        pub fn hummock_time_travel_filter_out_objects_v1() -> bool {
319            false
320        }
321
322        pub fn hummock_time_travel_filter_out_objects_list_version_batch_size() -> usize {
323            10
324        }
325
326        pub fn hummock_time_travel_filter_out_objects_list_delta_batch_size() -> usize {
327            1000
328        }
329
330        pub fn memory_controller_threshold_aggressive() -> f64 {
331            0.9
332        }
333
334        pub fn memory_controller_threshold_graceful() -> f64 {
335            0.81
336        }
337
338        pub fn memory_controller_threshold_stable() -> f64 {
339            0.72
340        }
341
342        pub fn memory_controller_eviction_factor_aggressive() -> f64 {
343            2.0
344        }
345
346        pub fn memory_controller_eviction_factor_graceful() -> f64 {
347            1.5
348        }
349
350        pub fn memory_controller_eviction_factor_stable() -> f64 {
351            1.0
352        }
353
354        pub fn memory_controller_update_interval_ms() -> usize {
355            100
356        }
357
358        pub fn memory_controller_sequence_tls_step() -> u64 {
359            128
360        }
361
362        pub fn memory_controller_sequence_tls_lag() -> u64 {
363            32
364        }
365
366        pub fn stream_enable_arrangement_backfill() -> bool {
367            true
368        }
369
370        pub fn stream_enable_snapshot_backfill() -> bool {
371            true
372        }
373
374        pub fn enable_shared_source() -> bool {
375            true
376        }
377
378        pub fn stream_high_join_amplification_threshold() -> usize {
379            2048
380        }
381
382        pub fn stream_high_gap_fill_amplification_threshold() -> usize {
383            2048
384        }
385
386        /// Default to 1 to be compatible with the behavior before this config is introduced.
387        pub fn stream_exchange_connection_pool_size() -> Option<u16> {
388            Some(1)
389        }
390
391        pub fn enable_actor_tokio_metrics() -> bool {
392            true
393        }
394
395        pub fn stream_enable_auto_schema_change() -> bool {
396            true
397        }
398
399        pub fn switch_jdbc_pg_to_native() -> bool {
400            false
401        }
402
403        pub fn streaming_hash_join_entry_state_max_rows() -> usize {
404            // NOTE(kwannoel): This is just an arbitrary number.
405            30000
406        }
407
408        pub fn streaming_join_hash_map_evict_interval_rows() -> u32 {
409            16
410        }
411
412        pub fn streaming_now_progress_ratio() -> Option<f32> {
413            None
414        }
415
416        pub fn stream_snapshot_iter_rebuild_interval_secs() -> u64 {
417            10 * 60
418        }
419
420        pub fn enable_explain_analyze_stats() -> bool {
421            true
422        }
423
424        pub fn rpc_client_connect_timeout_secs() -> u64 {
425            5
426        }
427
428        pub fn rpc_client_pool_setup_concurrency() -> usize {
429            0
430        }
431
432        pub fn iceberg_list_interval_sec() -> u64 {
433            10
434        }
435
436        pub fn iceberg_fetch_batch_size() -> u64 {
437            1024
438        }
439
440        pub fn iceberg_update_ordered_max_tasks() -> u64 {
441            16
442        }
443
444        pub fn iceberg_sink_positional_delete_cache_size() -> usize {
445            1024
446        }
447
448        pub fn iceberg_sink_write_parquet_max_row_group_rows() -> usize {
449            100_000
450        }
451
452        pub fn iceberg_pk_index_sink_lookup_concurrency() -> usize {
453            64
454        }
455
456        pub fn iceberg_pk_index_sink_max_batch_rows() -> usize {
457            4096
458        }
459
460        pub fn materialize_force_overwrite_on_no_check() -> bool {
461            false
462        }
463
464        pub fn refresh_scheduler_interval_sec() -> u64 {
465            60
466        }
467
468        pub fn sync_log_store_pause_duration_ms() -> usize {
469            64
470        }
471
472        pub fn sync_log_store_buffer_size() -> usize {
473            2048
474        }
475
476        pub fn disable_sync_log_store_dispatcher() -> bool {
477            false
478        }
479
480        pub fn table_change_log_insert_batch_size() -> u64 {
481            1000
482        }
483
484        pub fn table_change_log_delete_batch_size() -> u64 {
485            1000
486        }
487
488        pub fn table_change_log_truncate_interval_sec() -> u64 {
489            600
490        }
491
492        pub fn enable_state_table_vnode_stats_pruning() -> bool {
493            false
494        }
495
496        pub fn enable_locality_sort_buffer() -> bool {
497            true
498        }
499
500        pub fn locality_sort_buffer_activate_threshold() -> u64 {
501            65536
502        }
503
504        pub fn cache_refill_policy() -> CacheRefillPolicy {
505            CacheRefillPolicy::Enabled
506        }
507
508        pub fn enable_vnode_key_stats_for_materialize() -> bool {
509            false
510        }
511
512        pub fn max_concurrent_kv_log_store_historical_read() -> usize {
513            0
514        }
515    }
516}
517
518pub const MAX_META_CACHE_SHARD_BITS: usize = 4;
519pub const MIN_BUFFER_SIZE_PER_SHARD: usize = 256;
520pub const MAX_BLOCK_CACHE_SHARD_BITS: usize = 6; // It means that there will be 64 shards lru-cache to avoid lock conflict.
521
522#[cfg(test)]
523pub mod tests {
524    use expect_test::expect;
525    use risingwave_license::LicenseKey;
526
527    use super::*;
528
529    fn default_config_for_docs() -> RwConfig {
530        let mut config = RwConfig::default();
531        // Set `license_key` to empty in the docs to avoid any confusion.
532        config.system.license_key = Some(LicenseKey::empty());
533        // Keep generated docs and example config aligned with the production-safe default.
534        config.frontend.unsafe_enable_local_fs_connector = false;
535        config
536    }
537
538    /// This test ensures that `config/example.toml` is up-to-date with the default values specified
539    /// in this file. Developer should run `./risedev generate-example-config` to update it if this
540    /// test fails.
541    #[test]
542    fn test_example_up_to_date() {
543        const HEADER: &str = "# This file is generated by ./risedev generate-example-config
544# Check detailed comments in src/common/src/config.rs";
545
546        let actual = expect_test::expect_file!["../../../config/example.toml"];
547        let default = toml::to_string(&default_config_for_docs()).expect("failed to serialize");
548
549        let expected = format!("{HEADER}\n\n{default}");
550        actual.assert_eq(&expected);
551
552        let expected = rw_config_to_markdown();
553        let actual = expect_test::expect_file!["../../../config/docs.md"];
554        actual.assert_eq(&expected);
555    }
556
557    #[test]
558    fn test_sql_option_redaction_allowlist_config() {
559        let config: RwConfig = toml::from_str("").unwrap();
560        assert_eq!(config.batch.redact_sql_option_allowlist, ["primary_key"]);
561
562        let config: RwConfig = toml::from_str("[batch]\nredact_sql_option_allowlist = []").unwrap();
563        assert!(config.batch.redact_sql_option_allowlist.is_empty());
564
565        let config: RwConfig =
566            toml::from_str("[batch]\nredact_sql_option_allowlist = ['partition_key']").unwrap();
567        assert_eq!(config.batch.redact_sql_option_allowlist, ["partition_key"]);
568    }
569
570    #[test]
571    fn test_session_init_entries_distinguishes_omitted_from_default() {
572        let config: RwConfig = toml::from_str(
573            r#"
574            [session_init]
575            streaming_parallelism = "bounded(8)"
576            streaming_parallelism_for_table = "default"
577            "#,
578        )
579        .unwrap();
580
581        // Omitted fields are `None`; an explicit `default` is `Some("default")`.
582        assert_eq!(
583            config.session_init.streaming_parallelism.as_deref(),
584            Some("bounded(8)")
585        );
586        assert_eq!(
587            config
588                .session_init
589                .streaming_parallelism_for_table
590                .as_deref(),
591            Some("default")
592        );
593        assert_eq!(config.session_init.streaming_parallelism_for_sink, None);
594
595        // Only explicitly-configured parameters are reported, by their session parameter name.
596        assert_eq!(
597            config.session_init.entries(),
598            vec![
599                ("streaming_parallelism", "bounded(8)"),
600                ("streaming_parallelism_for_table", "default"),
601            ]
602        );
603    }
604
605    #[test]
606    fn test_session_init_rejects_unrecognized_key() {
607        let err = toml::from_str::<RwConfig>(
608            r#"
609            [session_init]
610            streaming_parallelism = "bounded(8)"
611            not_a_real_param = "oops"
612            "#,
613        )
614        .unwrap_err();
615
616        assert!(err.to_string().contains("unknown field `not_a_real_param`"));
617    }
618
619    #[derive(Debug)]
620    struct ConfigItemDoc {
621        desc: String,
622        default: String,
623    }
624
625    fn rw_config_to_markdown() -> String {
626        let mut config_rustdocs = BTreeMap::<String, Vec<(String, String)>>::new();
627        RwConfig::config_docs("".to_owned(), &mut config_rustdocs);
628
629        // Section -> Config Name -> ConfigItemDoc
630        let mut configs: BTreeMap<String, BTreeMap<String, ConfigItemDoc>> = config_rustdocs
631            .into_iter()
632            .map(|(k, v)| {
633                let docs: BTreeMap<String, ConfigItemDoc> = v
634                    .into_iter()
635                    .map(|(name, desc)| {
636                        (
637                            name,
638                            ConfigItemDoc {
639                                desc,
640                                default: "".to_owned(), // unset
641                            },
642                        )
643                    })
644                    .collect();
645                (k, docs)
646            })
647            .collect();
648
649        let toml_doc: BTreeMap<String, toml::Value> =
650            toml::from_str(&toml::to_string(&default_config_for_docs()).unwrap()).unwrap();
651        toml_doc.into_iter().for_each(|(name, value)| {
652            set_default_values("".to_owned(), name, value, &mut configs);
653        });
654
655        let mut markdown = "# RisingWave System Configurations\n\n".to_owned()
656            + "This page is automatically generated by `./risedev generate-example-config`\n";
657        for (section, configs) in configs {
658            if configs.is_empty() {
659                continue;
660            }
661            markdown.push_str(&format!("\n## {}\n\n", section));
662            markdown.push_str("| Config | Description | Default |\n");
663            markdown.push_str("|--------|-------------|---------|\n");
664            for (config, doc) in configs {
665                markdown.push_str(&format!(
666                    "| {} | {} | {} |\n",
667                    config, doc.desc, doc.default
668                ));
669            }
670        }
671        markdown
672    }
673
674    fn set_default_values(
675        section: String,
676        name: String,
677        value: toml::Value,
678        configs: &mut BTreeMap<String, BTreeMap<String, ConfigItemDoc>>,
679    ) {
680        // Set the default value if it's a config name-value pair, otherwise it's a sub-section (Table) that should be recursively processed.
681        if let toml::Value::Table(table) = value {
682            let section_configs: BTreeMap<String, toml::Value> = table.into_iter().collect();
683            let sub_section = if section.is_empty() {
684                name
685            } else {
686                format!("{}.{}", section, name)
687            };
688            section_configs
689                .into_iter()
690                .for_each(|(k, v)| set_default_values(sub_section.clone(), k, v, configs))
691        } else if let Some(t) = configs.get_mut(&section)
692            && let Some(item_doc) = t.get_mut(&name)
693        {
694            item_doc.default = format!("{}", value);
695        }
696    }
697
698    #[test]
699    fn test_opendal_chunked_streaming_read_config() {
700        let config: ObjectStoreConfig = toml::from_str("").unwrap();
701        assert_eq!(config.opendal_streaming_read_chunk_size, 0);
702        assert_eq!(config.opendal_streaming_read_chunk_concurrency, 1);
703
704        let config: ObjectStoreConfig = toml::from_str(
705            "opendal_streaming_read_chunk_size = 1024\nopendal_streaming_read_chunk_concurrency = 4",
706        )
707        .unwrap();
708        assert_eq!(config.opendal_streaming_read_chunk_size, 1024);
709        assert_eq!(config.opendal_streaming_read_chunk_concurrency, 4);
710
711        let config: ObjectStoreConfig =
712            toml::from_str("opendal_streaming_read_chunk_size = 0").unwrap();
713        assert_eq!(config.opendal_streaming_read_chunk_size, 0);
714    }
715
716    #[test]
717    fn test_http_client_rotation_config() {
718        for (input, expected) in [
719            ("", None),
720            ("http_client_rotation_interval_ms = \"\"", None),
721            ("http_client_rotation_interval_ms = 60000", Some(60000)),
722        ] {
723            let config: RwConfig =
724                toml::from_str(&format!("[storage.object_store.s3]\n{input}")).unwrap();
725            assert_eq!(
726                config
727                    .storage
728                    .object_store
729                    .s3
730                    .http_client_rotation_interval_ms,
731                expected
732            );
733            let roundtrip: RwConfig = toml::from_str(&toml::to_string(&config).unwrap()).unwrap();
734            assert_eq!(
735                roundtrip
736                    .storage
737                    .object_store
738                    .s3
739                    .http_client_rotation_interval_ms,
740                expected
741            );
742        }
743        let error = toml::from_str::<RwConfig>(
744            "[storage.object_store.s3]\nhttp_client_rotation_interval_ms = 0",
745        )
746        .unwrap_err();
747        assert!(error.to_string().contains("must be greater than 0"));
748    }
749
750    #[test]
751    fn test_object_store_configs_backward_compatibility() {
752        // Define configs with the old name and make sure it still works
753        {
754            let config: RwConfig = toml::from_str(
755                r#"
756            [storage.object_store]
757            object_store_set_atomic_write_dir = true
758
759            [storage.object_store.s3]
760            object_store_keepalive_ms = 1
761            object_store_send_buffer_size = 1
762            object_store_recv_buffer_size = 1
763            object_store_nodelay = false
764
765            [storage.object_store.s3.developer]
766            object_store_retry_unknown_service_error = true
767            object_store_retryable_service_error_codes = ['dummy']
768
769
770            "#,
771            )
772            .unwrap();
773
774            assert!(config.storage.object_store.set_atomic_write_dir);
775            assert_eq!(config.storage.object_store.s3.keepalive_ms, Some(1));
776            assert_eq!(config.storage.object_store.s3.send_buffer_size, Some(1));
777            assert_eq!(config.storage.object_store.s3.recv_buffer_size, Some(1));
778            assert_eq!(config.storage.object_store.s3.nodelay, Some(false));
779            assert!(
780                config
781                    .storage
782                    .object_store
783                    .s3
784                    .developer
785                    .retry_unknown_service_error
786            );
787            assert_eq!(
788                config
789                    .storage
790                    .object_store
791                    .s3
792                    .developer
793                    .retryable_service_error_codes,
794                vec!["dummy".to_owned()]
795            );
796        }
797
798        // Define configs with the new name and make sure it works
799        {
800            let config: RwConfig = toml::from_str(
801                r#"
802            [storage.object_store]
803            set_atomic_write_dir = true
804
805            [storage.object_store.s3]
806            keepalive_ms = 1
807            send_buffer_size = 1
808            recv_buffer_size = 1
809            nodelay = false
810
811            [storage.object_store.s3.developer]
812            retry_unknown_service_error = true
813            retryable_service_error_codes = ['dummy']
814
815
816            "#,
817            )
818            .unwrap();
819
820            assert!(config.storage.object_store.set_atomic_write_dir);
821            assert_eq!(config.storage.object_store.s3.keepalive_ms, Some(1));
822            assert_eq!(config.storage.object_store.s3.send_buffer_size, Some(1));
823            assert_eq!(config.storage.object_store.s3.recv_buffer_size, Some(1));
824            assert_eq!(config.storage.object_store.s3.nodelay, Some(false));
825            assert!(
826                config
827                    .storage
828                    .object_store
829                    .s3
830                    .developer
831                    .retry_unknown_service_error
832            );
833            assert_eq!(
834                config
835                    .storage
836                    .object_store
837                    .s3
838                    .developer
839                    .retryable_service_error_codes,
840                vec!["dummy".to_owned()]
841            );
842        }
843    }
844
845    #[test]
846    fn test_file_cache_separated_runtime_config_backward_compatibility() {
847        let config: RwConfig = toml::from_str(
848            r#"
849            [storage.data_file_cache.runtime_config.Separated.read_runtime_options]
850            worker_threads = 2
851            max_blocking_threads = 4
852
853            [storage.data_file_cache.runtime_config.Separated.write_runtime_options]
854            worker_threads = 6
855            max_blocking_threads = 8
856            "#,
857        )
858        .unwrap();
859
860        let storage::FileCacheRuntimeConfig::Separated {
861            read_runtime_options,
862            write_runtime_options,
863        } = config.storage.data_file_cache.runtime_config
864        else {
865            panic!("expected legacy separated file-cache runtime config");
866        };
867        assert_eq!(read_runtime_options.worker_threads, 2);
868        assert_eq!(read_runtime_options.max_blocking_threads, 4);
869        assert_eq!(write_runtime_options.worker_threads, 6);
870        assert_eq!(write_runtime_options.max_blocking_threads, 8);
871    }
872
873    #[test]
874    fn test_meta_configs_backward_compatibility() {
875        // Test periodic_space_reclaim_compaction_interval_sec
876        {
877            let config: RwConfig = toml::from_str(
878                r#"
879            [meta]
880            periodic_split_compact_group_interval_sec = 1
881            table_write_throughput_threshold = 10
882            min_table_split_write_throughput = 5
883            "#,
884            )
885            .unwrap();
886
887            assert_eq!(
888                config
889                    .meta
890                    .periodic_scheduling_compaction_group_split_interval_sec,
891                1
892            );
893            assert_eq!(config.meta.table_high_write_throughput_threshold, 10);
894            assert_eq!(config.meta.table_low_write_throughput_threshold, 5);
895        }
896    }
897
898    #[test]
899    fn test_meta_max_normalize_splits_per_round_must_be_positive() {
900        let config = toml::from_str::<RwConfig>(
901            r#"
902            [meta]
903            max_normalize_splits_per_round = 0
904            "#,
905        )
906        .unwrap_err();
907
908        expect![[r#"
909            TOML parse error at line 3, column 46
910              |
911            3 |             max_normalize_splits_per_round = 0
912              |                                              ^
913            meta.max_normalize_splits_per_round must be greater than 0
914        "#]]
915        .assert_eq(&config.to_string());
916    }
917
918    // Previously, we have prefixes like `stream_` for all configs under `streaming.developer`.
919    // Later we removed the prefixes, but we still want to guarantee the backward compatibility.
920    #[test]
921    fn test_prefix_alias() {
922        let config: RwConfig = toml::from_str(
923            "
924            [streaming.developer]
925            stream_chunk_size = 114514
926
927            [streaming.developer.stream_compute_client_config]
928            connect_timeout_secs = 42
929            pool_setup_concurrency = 10
930            ",
931        )
932        .unwrap();
933
934        assert_eq!(config.streaming.developer.chunk_size, 114514);
935        assert_eq!(
936            config
937                .streaming
938                .developer
939                .compute_client_config
940                .connect_timeout_secs,
941            42
942        );
943        assert_eq!(
944            config
945                .streaming
946                .developer
947                .compute_client_config
948                .pool_setup_concurrency,
949            10
950        );
951    }
952
953    #[test]
954    fn test_prefix_alias_duplicate() {
955        let config = toml::from_str::<RwConfig>(
956            "
957            [streaming.developer]
958            stream_chunk_size = 114514
959            chunk_size = 1919810
960            ",
961        )
962        .unwrap_err();
963
964        expect![[r#"
965            TOML parse error at line 2, column 13
966              |
967            2 |             [streaming.developer]
968              |             ^^^^^^^^^^^^^^^^^^^^^
969            duplicate field `chunk_size`
970        "#]]
971        .assert_eq(&config.to_string());
972
973        let config = toml::from_str::<RwConfig>(
974            "
975            [streaming.developer.stream_compute_client_config]
976            connect_timeout_secs = 5
977
978            [streaming.developer.compute_client_config]
979            connect_timeout_secs = 10
980            ",
981        )
982        .unwrap_err();
983
984        expect![[r#"
985            TOML parse error at line 2, column 24
986              |
987            2 |             [streaming.developer.stream_compute_client_config]
988              |                        ^^^^^^^^^
989            duplicate field `compute_client_config`
990        "#]]
991        .assert_eq(&config.to_string());
992    }
993
994    #[test]
995    fn test_storage_sstable_meta_fetch_concurrency() {
996        let default: RwConfig = toml::from_str("").unwrap();
997        assert_eq!(default.storage.sstable_meta_fetch_concurrency, 1);
998        let config: RwConfig =
999            toml::from_str("[storage]\nsstable_meta_fetch_concurrency = 8").unwrap();
1000        assert_eq!(config.storage.sstable_meta_fetch_concurrency, 8);
1001        let error = toml::from_str::<RwConfig>("[storage]\nsstable_meta_fetch_concurrency = 0")
1002            .unwrap_err();
1003        assert!(
1004            error
1005                .to_string()
1006                .contains("storage.sstable_meta_fetch_concurrency must be greater than 0")
1007        );
1008    }
1009
1010    #[test]
1011    fn test_storage_max_prefetch_block_number_must_be_positive() {
1012        let config = toml::from_str::<RwConfig>(
1013            r#"
1014            [storage]
1015            max_prefetch_block_number = 0
1016            "#,
1017        )
1018        .unwrap_err();
1019
1020        expect![[r#"
1021            TOML parse error at line 3, column 41
1022              |
1023            3 |             max_prefetch_block_number = 0
1024              |                                         ^
1025            storage.max_prefetch_block_number must be greater than 0
1026        "#]]
1027        .assert_eq(&config.to_string());
1028    }
1029
1030    #[test]
1031    fn test_storage_iceberg_compaction_pull_interval_ms_must_be_positive() {
1032        let config = toml::from_str::<RwConfig>(
1033            r#"
1034            [storage]
1035            iceberg_compaction_pull_interval_ms = 0
1036            "#,
1037        )
1038        .unwrap_err();
1039
1040        expect![[r#"
1041            TOML parse error at line 3, column 51
1042              |
1043            3 |             iceberg_compaction_pull_interval_ms = 0
1044              |                                                   ^
1045            storage.iceberg_compaction_pull_interval_ms must be greater than 0
1046        "#]]
1047        .assert_eq(&config.to_string());
1048    }
1049}