risingwave_common/config/
merge.rs1use std::str::FromStr as _;
16
17use anyhow::Context as _;
18use serde::Serialize;
19use serde::de::DeserializeOwned;
20use toml::map::Entry;
21use toml::{Table, Value};
22
23use crate::config::StreamingConfig;
24
25def_anyhow_newtype! { pub ConfigMergeError }
26
27pub fn merge_config<C: Serialize + DeserializeOwned + Clone>(
35 base: &C,
36 partial: &str,
37 partial_path: impl IntoIterator<Item = &str>,
38) -> Result<Option<C>, ConfigMergeError> {
39 let partial_table = {
40 let mut partial_table =
41 Table::from_str(partial).context("failed to parse partial config")?;
42 for k in partial_path {
43 if let Some(v) = partial_table.remove(k)
44 && let Value::Table(t) = v
45 {
46 partial_table = t;
47 } else {
48 return Ok(None);
50 }
51 }
52 partial_table
53 };
54
55 if partial_table.is_empty() {
56 return Ok(None);
58 }
59
60 let mut base_table = Table::try_from(base).context("failed to serialize base config")?;
61
62 fn merge_table(base_table: &mut Table, partial_table: Table) {
63 for (k, v) in partial_table {
64 match base_table.entry(k) {
65 Entry::Vacant(entry) => {
66 entry.insert(v);
69 }
70 Entry::Occupied(mut entry) => {
71 let base_v = entry.get_mut();
72 merge_value(base_v, v);
73 }
74 }
75 }
76 }
77
78 fn merge_value(base: &mut Value, partial: Value) {
79 if let Value::Table(base_table) = base
80 && let Value::Table(partial_table) = partial
81 {
82 merge_table(base_table, partial_table);
83 } else {
84 *base = partial;
86 }
87 }
88
89 merge_table(&mut base_table, partial_table);
90
91 let merged: C = base_table
92 .try_into()
93 .context("failed to deserialize merged config")?;
94
95 Ok(Some(merged))
96}
97
98pub fn merge_streaming_config_section(
102 base: &StreamingConfig,
103 partial: &str,
104) -> Result<Option<StreamingConfig>, ConfigMergeError> {
105 merge_config(base, partial, ["streaming"])
106}
107
108#[cfg(test)]
109#[expect(clippy::bool_assert_comparison)]
110mod tests {
111 use thiserror_ext::AsReport;
112
113 use super::*;
114 use crate::config::StreamingConfig;
115
116 #[test]
117 fn test_merge_streaming_config() {
118 let base = StreamingConfig::default();
119 assert_eq!(base.unsafe_disable_strict_consistency, false);
120 assert_ne!(base.developer.chunk_size, 114514);
121 assert_ne!(
122 base.developer.compute_client_config.connect_timeout_secs,
123 114514
124 );
125
126 let partial = r#"
127 [streaming]
128 unsafe_disable_strict_consistency = true
129
130 [streaming.developer]
131 chunk_size = 114514
132 compute_client_config = { connect_timeout_secs = 114514 }
133 "#;
134 let merged = merge_streaming_config_section(&base, partial)
135 .unwrap()
136 .unwrap();
137
138 assert_eq!(merged.unsafe_disable_strict_consistency, true);
140 assert_eq!(merged.developer.chunk_size, 114514);
141 assert_eq!(
142 merged.developer.compute_client_config.connect_timeout_secs,
143 114514
144 );
145
146 {
148 let mut merged = merged;
149 merged.unsafe_disable_strict_consistency = base.unsafe_disable_strict_consistency;
150 merged.developer.chunk_size = base.developer.chunk_size;
151 merged.developer.compute_client_config.connect_timeout_secs =
152 base.developer.compute_client_config.connect_timeout_secs;
153
154 pretty_assertions::assert_eq!(format!("{base:?}"), format!("{merged:?}"));
155 }
156 }
157
158 #[test]
159 fn test_projection_job_overrides_are_independent() {
160 let mut base = StreamingConfig::default();
161 base.developer.project_expr_concurrency = 3;
162 base.developer.project_expr_inflight_request_concurrency = 20;
163 let job_a = merge_streaming_config_section(
164 &base,
165 "[streaming.developer]\nproject_expr_concurrency = 7\nproject_expr_inflight_request_concurrency = 0",
166 )
167 .unwrap()
168 .unwrap();
169 let job_b = merge_streaming_config_section(
170 &base,
171 "[streaming.developer]\nproject_expr_inflight_request_concurrency = 5",
172 )
173 .unwrap()
174 .unwrap();
175 assert_eq!(job_a.developer.project_expr_concurrency, 7);
176 assert_eq!(job_a.developer.project_expr_inflight_request_concurrency, 0);
177 assert_eq!(job_b.developer.project_expr_concurrency, 3);
178 assert_eq!(job_b.developer.project_expr_inflight_request_concurrency, 5);
179 assert_eq!(base.developer.project_expr_concurrency, 3);
180 assert_eq!(base.developer.project_expr_inflight_request_concurrency, 20);
181 }
182
183 #[test]
184 fn test_not_relevant() {
185 let base = StreamingConfig::default();
186 let partial = r#"
187 [batch.developer]
188 chunk_size = 114514
189 "#;
190 let merged = merge_streaming_config_section(&base, partial).unwrap();
191 assert!(
192 merged.is_none(),
193 "nothing to override, but got: {merged:#?}"
194 );
195 }
196
197 #[test]
198 fn test_nothing_to_override() {
199 let base = StreamingConfig::default();
200 let partial = r#"
201 [streaming]
202 "#;
203 let merged = merge_streaming_config_section(&base, partial).unwrap();
204 assert!(
205 merged.is_none(),
206 "nothing to override, but got: {merged:#?}"
207 );
208 }
209
210 #[test]
211 fn test_unrecognized_entry() {
212 let base = StreamingConfig::default();
213 let partial = r#"
214 [streaming]
215 no_such_entry = 114514
216
217 [streaming.developer]
218 no_such_dev_entry = 1919810
219 "#;
220 let merged = merge_streaming_config_section(&base, partial)
221 .unwrap()
222 .unwrap();
223
224 let unrecognized = merged.unrecognized.into_inner();
225 assert_eq!(unrecognized.len(), 1);
226 assert_eq!(unrecognized["no_such_entry"], 114514);
227
228 let dev_unrecognized = merged.developer.unrecognized.into_inner();
229 assert_eq!(dev_unrecognized.len(), 1);
230 assert_eq!(dev_unrecognized["no_such_dev_entry"], 1919810);
231 }
232
233 #[test]
234 fn test_invalid_type() {
235 let base = StreamingConfig::default();
236 let partial = r#"
237 [streaming.developer]
238 chunk_size = "omakase"
239 "#;
240 let error = merge_streaming_config_section(&base, partial).unwrap_err();
241 expect_test::expect![[r#"
242 failed to deserialize merged config: invalid type: string "omakase", expected usize
243 in `developer.chunk_size`
244 "#]]
245 .assert_eq(&error.to_report_string());
246 }
247
248 #[test]
253 fn test_override_with_legacy_prefixed_config() {
254 let base = StreamingConfig::default();
255 let partial = r#"
256 [streaming.developer]
257 stream_chunk_size = 114514
258 "#;
259 let error = merge_streaming_config_section(&base, partial).unwrap_err();
260 expect_test::expect![[r#"
261 failed to deserialize merged config: duplicate field `chunk_size`
262 in `developer`
263 "#]]
264 .assert_eq(&error.to_report_string());
265 }
266
267 #[test]
269 fn test_some_back_to_none() {
270 assert!(
271 StreamingConfig::default()
272 .actor_runtime_worker_threads_num
273 .is_none()
274 );
275
276 let base = toml::from_str(
277 r#"
278 [streaming]
279 actor_runtime_worker_threads_num = 114514
280 "#,
281 )
282 .unwrap();
283 let partial = r#"
284 [streaming]
285 actor_runtime_worker_threads_num = ""
286 "#;
287
288 let merged = merge_streaming_config_section(&base, partial)
289 .unwrap()
290 .unwrap();
291 assert!(merged.actor_runtime_worker_threads_num.is_none());
292 }
293}