Skip to main content

risingwave_common/config/
merge.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::str::FromStr as _;
16
17use anyhow::Context as _;
18use serde::Serialize;
19use serde::de::DeserializeOwned;
20use toml::map::Entry;
21use toml::{Table, Value};
22
23use crate::config::StreamingConfig;
24
25def_anyhow_newtype! { pub ConfigMergeError }
26
27/// Extract the section at `partial_path` from `partial`, merge it into `base` to override entries.
28///
29/// Tables will be merged recursively, while other fields (including arrays) will be replaced by
30/// the `partial` config, if exists.
31///
32/// Returns an error if any of the input is invalid, or the merged config cannot be parsed.
33/// Returns `None` if there's nothing to override.
34pub fn merge_config<C: Serialize + DeserializeOwned + Clone>(
35    base: &C,
36    partial: &str,
37    partial_path: impl IntoIterator<Item = &str>,
38) -> Result<Option<C>, ConfigMergeError> {
39    let partial_table = {
40        let mut partial_table =
41            Table::from_str(partial).context("failed to parse partial config")?;
42        for k in partial_path {
43            if let Some(v) = partial_table.remove(k)
44                && let Value::Table(t) = v
45            {
46                partial_table = t;
47            } else {
48                // The section to override is not relevant.
49                return Ok(None);
50            }
51        }
52        partial_table
53    };
54
55    if partial_table.is_empty() {
56        // Nothing to override.
57        return Ok(None);
58    }
59
60    let mut base_table = Table::try_from(base).context("failed to serialize base config")?;
61
62    fn merge_table(base_table: &mut Table, partial_table: Table) {
63        for (k, v) in partial_table {
64            match base_table.entry(k) {
65                Entry::Vacant(entry) => {
66                    // Unrecognized entry might be tolerated.
67                    // So we simply keep it and defer the error (if any) until final deserialization.
68                    entry.insert(v);
69                }
70                Entry::Occupied(mut entry) => {
71                    let base_v = entry.get_mut();
72                    merge_value(base_v, v);
73                }
74            }
75        }
76    }
77
78    fn merge_value(base: &mut Value, partial: Value) {
79        if let Value::Table(base_table) = base
80            && let Value::Table(partial_table) = partial
81        {
82            merge_table(base_table, partial_table);
83        } else {
84            // We don't validate the type, but defer until final deserialization.
85            *base = partial;
86        }
87    }
88
89    merge_table(&mut base_table, partial_table);
90
91    let merged: C = base_table
92        .try_into()
93        .context("failed to deserialize merged config")?;
94
95    Ok(Some(merged))
96}
97
98/// Extract the `streaming` section from `partial`, merge it into `base` to override entries.
99///
100/// See [`merge_config`] for more details.
101pub fn merge_streaming_config_section(
102    base: &StreamingConfig,
103    partial: &str,
104) -> Result<Option<StreamingConfig>, ConfigMergeError> {
105    merge_config(base, partial, ["streaming"])
106}
107
108#[cfg(test)]
109#[expect(clippy::bool_assert_comparison)]
110mod tests {
111    use thiserror_ext::AsReport;
112
113    use super::*;
114    use crate::config::StreamingConfig;
115
116    #[test]
117    fn test_merge_streaming_config() {
118        let base = StreamingConfig::default();
119        assert_eq!(base.unsafe_disable_strict_consistency, false);
120        assert_ne!(base.developer.chunk_size, 114514);
121        assert_ne!(
122            base.developer.compute_client_config.connect_timeout_secs,
123            114514
124        );
125
126        let partial = r#"
127            [streaming]
128            unsafe_disable_strict_consistency = true
129
130            [streaming.developer]
131            chunk_size = 114514
132            compute_client_config = { connect_timeout_secs = 114514 }
133        "#;
134        let merged = merge_streaming_config_section(&base, partial)
135            .unwrap()
136            .unwrap();
137
138        // Demonstrate that the entries are merged.
139        assert_eq!(merged.unsafe_disable_strict_consistency, true);
140        assert_eq!(merged.developer.chunk_size, 114514);
141        assert_eq!(
142            merged.developer.compute_client_config.connect_timeout_secs,
143            114514
144        );
145
146        // Demonstrate that the rest of the config is not affected.
147        {
148            let mut merged = merged;
149            merged.unsafe_disable_strict_consistency = base.unsafe_disable_strict_consistency;
150            merged.developer.chunk_size = base.developer.chunk_size;
151            merged.developer.compute_client_config.connect_timeout_secs =
152                base.developer.compute_client_config.connect_timeout_secs;
153
154            pretty_assertions::assert_eq!(format!("{base:?}"), format!("{merged:?}"));
155        }
156    }
157
158    #[test]
159    fn test_projection_job_overrides_are_independent() {
160        let mut base = StreamingConfig::default();
161        base.developer.project_expr_concurrency = 3;
162        base.developer.project_expr_inflight_request_concurrency = 20;
163        let job_a = merge_streaming_config_section(
164            &base,
165            "[streaming.developer]\nproject_expr_concurrency = 7\nproject_expr_inflight_request_concurrency = 0",
166        )
167        .unwrap()
168        .unwrap();
169        let job_b = merge_streaming_config_section(
170            &base,
171            "[streaming.developer]\nproject_expr_inflight_request_concurrency = 5",
172        )
173        .unwrap()
174        .unwrap();
175        assert_eq!(job_a.developer.project_expr_concurrency, 7);
176        assert_eq!(job_a.developer.project_expr_inflight_request_concurrency, 0);
177        assert_eq!(job_b.developer.project_expr_concurrency, 3);
178        assert_eq!(job_b.developer.project_expr_inflight_request_concurrency, 5);
179        assert_eq!(base.developer.project_expr_concurrency, 3);
180        assert_eq!(base.developer.project_expr_inflight_request_concurrency, 20);
181    }
182
183    #[test]
184    fn test_not_relevant() {
185        let base = StreamingConfig::default();
186        let partial = r#"
187            [batch.developer]
188            chunk_size = 114514
189        "#;
190        let merged = merge_streaming_config_section(&base, partial).unwrap();
191        assert!(
192            merged.is_none(),
193            "nothing to override, but got: {merged:#?}"
194        );
195    }
196
197    #[test]
198    fn test_nothing_to_override() {
199        let base = StreamingConfig::default();
200        let partial = r#"
201            [streaming]
202        "#;
203        let merged = merge_streaming_config_section(&base, partial).unwrap();
204        assert!(
205            merged.is_none(),
206            "nothing to override, but got: {merged:#?}"
207        );
208    }
209
210    #[test]
211    fn test_unrecognized_entry() {
212        let base = StreamingConfig::default();
213        let partial = r#"
214            [streaming]
215            no_such_entry = 114514
216
217            [streaming.developer]
218            no_such_dev_entry = 1919810
219        "#;
220        let merged = merge_streaming_config_section(&base, partial)
221            .unwrap()
222            .unwrap();
223
224        let unrecognized = merged.unrecognized.into_inner();
225        assert_eq!(unrecognized.len(), 1);
226        assert_eq!(unrecognized["no_such_entry"], 114514);
227
228        let dev_unrecognized = merged.developer.unrecognized.into_inner();
229        assert_eq!(dev_unrecognized.len(), 1);
230        assert_eq!(dev_unrecognized["no_such_dev_entry"], 1919810);
231    }
232
233    #[test]
234    fn test_invalid_type() {
235        let base = StreamingConfig::default();
236        let partial = r#"
237            [streaming.developer]
238            chunk_size = "omakase"
239        "#;
240        let error = merge_streaming_config_section(&base, partial).unwrap_err();
241        expect_test::expect![[r#"
242            failed to deserialize merged config: invalid type: string "omakase", expected usize
243            in `developer.chunk_size`
244        "#]]
245        .assert_eq(&error.to_report_string());
246    }
247
248    // Even though we accept `stream_` prefixed config key when deserializing the config, since
249    // we perform merging atop of the raw `toml::Value`, we don't have the information about
250    // the aliasing. Therefore, using a prefixed config key in config override will result in
251    // a duplicate field error.
252    #[test]
253    fn test_override_with_legacy_prefixed_config() {
254        let base = StreamingConfig::default();
255        let partial = r#"
256            [streaming.developer]
257            stream_chunk_size = 114514
258        "#;
259        let error = merge_streaming_config_section(&base, partial).unwrap_err();
260        expect_test::expect![[r#"
261            failed to deserialize merged config: duplicate field `chunk_size`
262            in `developer`
263        "#]]
264        .assert_eq(&error.to_report_string());
265    }
266
267    // Show that by explicitly overriding to empty string, we can set a `Some` back to `None`.
268    #[test]
269    fn test_some_back_to_none() {
270        assert!(
271            StreamingConfig::default()
272                .actor_runtime_worker_threads_num
273                .is_none()
274        );
275
276        let base = toml::from_str(
277            r#"
278            [streaming]
279            actor_runtime_worker_threads_num = 114514
280        "#,
281        )
282        .unwrap();
283        let partial = r#"
284            [streaming]
285            actor_runtime_worker_threads_num = ""
286        "#;
287
288        let merged = merge_streaming_config_section(&base, partial)
289            .unwrap()
290            .unwrap();
291        assert!(merged.actor_runtime_worker_threads_num.is_none());
292    }
293}