risingwave_common/config/streaming.rs
1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use risingwave_common_proc_macro::serde_prefix_all;
18
19use super::*;
20
21mod async_stack_trace;
22mod cache_refill;
23mod join_encoding_type;
24mod over_window;
25
26pub use async_stack_trace::*;
27pub use cache_refill::*;
28pub use join_encoding_type::*;
29pub use over_window::*;
30
31/// The section `[streaming]` in `risingwave.toml`.
32#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
33#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
34pub struct StreamingConfig {
35 /// The maximum number of pending barriers in each partial graph. Pending barriers include
36 /// in-flight, collected but not committed, and currently completing barriers.
37 #[serde(default = "default::streaming::in_flight_barrier_nums")]
38 pub in_flight_barrier_nums: usize,
39
40 /// The maximum number of lagged barriers allowed when merging a snapshot backfill job into
41 /// the database graph.
42 #[serde(default = "default::streaming::snapshot_backfill_finish_max_lagged_barriers")]
43 pub snapshot_backfill_finish_max_lagged_barriers: usize,
44
45 /// The multiplier applied to `in_flight_barrier_nums` when limiting pending barriers in a
46 /// snapshot backfill partial graph. A value of 0 is treated as 1. The resulting limit must
47 /// be at least 2, so that queued catch-up barriers can still drain.
48 #[serde(default = "default::streaming::snapshot_backfill_barrier_amplification_factor")]
49 pub snapshot_backfill_barrier_amplification_factor: usize,
50
51 /// The thread number of the streaming actor runtime in the compute node. The default value is
52 /// decided by `tokio`.
53 #[serde(default)]
54 pub actor_runtime_worker_threads_num: Option<usize>,
55
56 /// Enable async stack tracing through `await-tree` for risectl.
57 #[serde(default = "default::streaming::async_stack_trace")]
58 pub async_stack_trace: AsyncStackTraceOption,
59
60 #[serde(default)]
61 #[config_doc(nested)]
62 pub developer: StreamingDeveloperConfig,
63
64 /// Max unique user stream errors per actor
65 #[serde(default = "default::streaming::unique_user_stream_errors")]
66 pub unique_user_stream_errors: usize,
67
68 /// Disable strict stream consistency checks.
69 #[serde(default = "default::streaming::unsafe_disable_strict_consistency")]
70 pub unsafe_disable_strict_consistency: bool,
71
72 #[serde(default, flatten)]
73 #[config_doc(omitted)]
74 pub unrecognized: Unrecognized<Self>,
75}
76
77/// The subsections `[streaming.developer]`.
78///
79/// It is put at [`StreamingConfig::developer`].
80#[serde_prefix_all("stream_", mode = "alias")]
81#[serde_with::apply(Option => #[serde(with = "none_as_empty_string")])]
82#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
83pub struct StreamingDeveloperConfig {
84 /// Set to true to enable per-executor row count metrics. This will produce a lot of timeseries
85 /// and might affect the prometheus performance. If you only need actor input and output
86 /// rows data, see `stream_actor_in_record_cnt` and `stream_actor_out_record_cnt` instead.
87 #[serde(default = "default::developer::stream_enable_executor_row_count")]
88 pub enable_executor_row_count: bool,
89
90 /// The capacity of the chunks in the channel that connects between `ConnectorSource` and
91 /// `SourceExecutor`.
92 #[serde(default = "default::developer::connector_message_buffer_size")]
93 pub connector_message_buffer_size: usize,
94
95 /// Limit number of the cached entries in an extreme aggregation call.
96 #[serde(default = "default::developer::unsafe_stream_extreme_cache_size")]
97 pub unsafe_extreme_cache_size: usize,
98
99 /// Minimum cache size for TopN cache per group key.
100 #[serde(default = "default::developer::stream_topn_cache_min_capacity")]
101 pub topn_cache_min_capacity: usize,
102
103 /// The maximum size of the chunk produced by executor at a time.
104 #[serde(default = "default::developer::stream_chunk_size")]
105 pub chunk_size: usize,
106
107 /// The initial permits that a channel holds, i.e., the maximum row count can be buffered in
108 /// the channel.
109 #[serde(default = "default::developer::stream_exchange_initial_permits")]
110 pub exchange_initial_permits: usize,
111
112 /// The permits that are batched to add back, for reducing the backward `AddPermits` messages
113 /// in remote exchange.
114 #[serde(default = "default::developer::stream_exchange_batched_permits")]
115 pub exchange_batched_permits: usize,
116
117 /// The maximum number of concurrent barriers in an exchange channel.
118 #[serde(default = "default::developer::stream_exchange_concurrent_barriers")]
119 pub exchange_concurrent_barriers: usize,
120
121 /// The concurrency for dispatching messages to different downstream jobs.
122 ///
123 /// - `1` means no concurrency, i.e., dispatch messages to downstream jobs one by one.
124 /// - `0` means unlimited concurrency.
125 #[serde(default = "default::developer::stream_exchange_concurrent_dispatchers")]
126 pub exchange_concurrent_dispatchers: usize,
127
128 /// The maximum number of chunks awaiting ordered output in an asynchronous `ProjectExecutor`.
129 /// Synchronous projections ignore this setting. Finished chunks waiting for earlier results
130 /// still occupy the window. This limit is per executor, not shared across a job's actors.
131 ///
132 /// - `1` means no chunk-level concurrency.
133 /// - `0` means unlimited concurrency.
134 #[serde(default = "default::developer::stream_project_expr_concurrency")]
135 pub project_expr_concurrency: usize,
136
137 /// The visible-row budget for in-flight evaluation in an asynchronous `ProjectExecutor`.
138 ///
139 /// A chunk charges its cardinality when evaluation starts and releases it when evaluation
140 /// finishes, even if its result is still waiting for ordered output. A whole chunk may enter
141 /// while usage is below the budget, exceeding it once; no further nonempty chunk enters until
142 /// usage falls below the budget. This lets chunks larger than the budget make progress.
143 /// Synchronous projections ignore this setting. This is a per-executor row budget, not an
144 /// exact HTTP-request limit: expressions may batch rows or issue multiple requests per row.
145 ///
146 /// - `0` means unlimited in-flight rows.
147 /// Both projection limits support job overrides with `ALTER ... SET CONFIG`; overrides take
148 /// precedence over the compute-node config and require job recovery to take effect.
149 #[serde(default = "default::developer::stream_project_expr_inflight_request_concurrency")]
150 pub project_expr_inflight_request_concurrency: usize,
151
152 /// The initial permits for a dml channel, i.e., the maximum row count can be buffered in
153 /// the channel.
154 #[serde(default = "default::developer::stream_dml_channel_initial_permits")]
155 pub dml_channel_initial_permits: usize,
156
157 /// The max heap size of dirty groups of `HashAggExecutor`.
158 #[serde(default = "default::developer::stream_hash_agg_max_dirty_groups_heap_size")]
159 pub hash_agg_max_dirty_groups_heap_size: usize,
160
161 #[serde(default = "default::developer::memory_controller_threshold_aggressive")]
162 pub memory_controller_threshold_aggressive: f64,
163
164 #[serde(default = "default::developer::memory_controller_threshold_graceful")]
165 pub memory_controller_threshold_graceful: f64,
166
167 #[serde(default = "default::developer::memory_controller_threshold_stable")]
168 pub memory_controller_threshold_stable: f64,
169
170 #[serde(default = "default::developer::memory_controller_eviction_factor_aggressive")]
171 pub memory_controller_eviction_factor_aggressive: f64,
172
173 #[serde(default = "default::developer::memory_controller_eviction_factor_graceful")]
174 pub memory_controller_eviction_factor_graceful: f64,
175
176 #[serde(default = "default::developer::memory_controller_eviction_factor_stable")]
177 pub memory_controller_eviction_factor_stable: f64,
178
179 #[serde(default = "default::developer::memory_controller_update_interval_ms")]
180 pub memory_controller_update_interval_ms: usize,
181
182 #[serde(default = "default::developer::memory_controller_sequence_tls_step")]
183 pub memory_controller_sequence_tls_step: u64,
184
185 #[serde(default = "default::developer::memory_controller_sequence_tls_lag")]
186 pub memory_controller_sequence_tls_lag: u64,
187
188 #[serde(default = "default::developer::stream_enable_arrangement_backfill")]
189 /// Deprecated and ignored for new streaming jobs. Arrangement backfill is always used as the
190 /// fallback backfill type.
191 #[deprecated(
192 note = "Deprecated and ignored for new streaming jobs. Arrangement backfill is always used as the fallback backfill type."
193 )]
194 pub enable_arrangement_backfill: bool,
195
196 #[serde(default = "default::developer::stream_enable_snapshot_backfill")]
197 /// Enable snapshot backfill
198 /// If false, the snapshot backfill will be disabled,
199 /// even if session variable set.
200 /// If true, it's decided by session variable `streaming_use_snapshot_backfill` (default true)
201 pub enable_snapshot_backfill: bool,
202
203 #[serde(default = "default::developer::stream_high_join_amplification_threshold")]
204 /// If number of hash join matches exceeds this threshold number,
205 /// it will be logged.
206 pub high_join_amplification_threshold: usize,
207
208 #[serde(default = "default::developer::stream_high_gap_fill_amplification_threshold")]
209 /// If number of rows generated by gap fill between two anchor rows exceeds this threshold
210 /// number, it will be logged.
211 pub high_gap_fill_amplification_threshold: usize,
212
213 /// Actor tokio metrics is enabled if `enable_actor_tokio_metrics` is set or metrics level >= Debug.
214 #[serde(default = "default::developer::enable_actor_tokio_metrics")]
215 pub enable_actor_tokio_metrics: bool,
216
217 /// The number of the connections for streaming remote exchange between two nodes.
218 /// If not specified, the value of `server.connection_pool_size` will be used.
219 #[serde(default = "default::developer::stream_exchange_connection_pool_size")]
220 pub(super) exchange_connection_pool_size: Option<u16>,
221
222 /// A flag to allow disabling the auto schema change handling
223 #[serde(default = "default::developer::stream_enable_auto_schema_change")]
224 pub enable_auto_schema_change: bool,
225
226 #[serde(default = "default::developer::enable_shared_source")]
227 /// Enable shared source
228 /// If false, the shared source will be disabled,
229 /// even if session variable set.
230 /// If true, it's decided by session variable `streaming_use_shared_source` (default true)
231 pub enable_shared_source: bool,
232
233 #[serde(default = "default::developer::switch_jdbc_pg_to_native")]
234 /// When true, all jdbc sinks with connector='jdbc' and jdbc.url="jdbc:postgresql://..."
235 /// will be switched from jdbc postgresql sinks to rust native (connector='postgres') sinks.
236 pub switch_jdbc_pg_to_native: bool,
237
238 /// The maximum number of consecutive barriers allowed in a message when sent between actors.
239 #[serde(default = "default::developer::stream_max_barrier_batch_size")]
240 pub max_barrier_batch_size: u32,
241
242 /// Configure the system-wide cache row cardinality of hash join.
243 /// For example, if this is set to 1000, it means we can have at most 1000 rows in cache.
244 #[serde(default = "default::developer::streaming_hash_join_entry_state_max_rows")]
245 pub hash_join_entry_state_max_rows: usize,
246
247 /// Number of processed rows between periodic join cache evictions.
248 /// Values smaller than 1 will be clamped to 1 by the executor.
249 #[serde(default = "default::developer::streaming_join_hash_map_evict_interval_rows")]
250 pub join_hash_map_evict_interval_rows: u32,
251
252 #[serde(default = "default::developer::streaming_now_progress_ratio")]
253 pub now_progress_ratio: Option<f32>,
254
255 /// Enable / Disable profiling stats used by `EXPLAIN ANALYZE`
256 #[serde(default = "default::developer::enable_explain_analyze_stats")]
257 pub enable_explain_analyze_stats: bool,
258
259 #[serde(default)]
260 pub compute_client_config: RpcClientConfig,
261
262 /// The interval in seconds to rebuild snapshot iterators during snapshot backfill.
263 #[serde(default = "default::developer::stream_snapshot_iter_rebuild_interval_secs")]
264 pub snapshot_iter_rebuild_interval_secs: u64,
265
266 /// `IcebergListExecutor`: The interval in seconds for Iceberg source to list new files.
267 #[serde(default = "default::developer::iceberg_list_interval_sec")]
268 pub iceberg_list_interval_sec: u64,
269
270 /// `IcebergFetchExecutor`: The number of files the executor will fetch concurrently in a batch.
271 #[serde(default = "default::developer::iceberg_fetch_batch_size")]
272 pub iceberg_fetch_batch_size: u64,
273
274 /// `IcebergUpdateListExecutor`: A `streaming_updates` phase with at most this many files
275 /// (capped by `iceberg_fetch_batch_size`) is fetched in order by a single Fetch actor,
276 /// without waiting for the previous phase to commit.
277 /// Larger phases are spread across Fetch actors and wait for their completion. Set to 0 to
278 /// always spread phases.
279 #[serde(default = "default::developer::iceberg_update_ordered_max_tasks")]
280 pub iceberg_update_ordered_max_tasks: u64,
281
282 /// `IcebergSink`: The size of the cache for positional delete in the sink.
283 #[serde(default = "default::developer::iceberg_sink_positional_delete_cache_size")]
284 pub iceberg_sink_positional_delete_cache_size: usize,
285
286 /// `IcebergSink`: The maximum number of rows in a row group when writing Parquet files.
287 #[serde(default = "default::developer::iceberg_sink_write_parquet_max_row_group_rows")]
288 pub iceberg_sink_write_parquet_max_row_group_rows: usize,
289
290 /// `IcebergWithPkIndexWriter`: The maximum number of concurrent PK index lookups issued for
291 /// deletes and updates in one batch. Values smaller than 1 will be clamped to 1.
292 #[serde(default = "default::developer::iceberg_pk_index_sink_lookup_concurrency")]
293 pub iceberg_pk_index_sink_lookup_concurrency: usize,
294
295 /// `IcebergWithPkIndexWriter`: The maximum number of input rows to process as one batch.
296 /// The writer only merges chunks that are already buffered in its input, so batching never
297 /// waits for new data. Set to 1 to process chunks one by one.
298 #[serde(default = "default::developer::iceberg_pk_index_sink_max_batch_rows")]
299 pub iceberg_pk_index_sink_max_batch_rows: usize,
300
301 /// When enabled, materialized views using default `NoCheck` conflict behavior will be forced
302 /// to use `Overwrite`. Useful to avoid propagating inconsistent changelog downstream.
303 #[serde(default = "default::developer::materialize_force_overwrite_on_no_check")]
304 pub materialize_force_overwrite_on_no_check: bool,
305
306 /// Whether by default enable preloading all rows in memory for state table.
307 /// If true, all capable state tables will preload its state to memory
308 #[serde(default = "default::streaming::default_enable_mem_preload_state_table")]
309 pub default_enable_mem_preload_state_table: bool,
310
311 /// The list of state table ids to *enable* preloading all rows in memory for state table.
312 /// Only takes effect when `default_enable_mem_preload_state_table` is false.
313 #[serde(default)]
314 pub mem_preload_state_table_ids_whitelist: Vec<u32>,
315
316 /// The list of state table ids to *disable* preloading all rows in memory for state table.
317 /// Only takes effect when `default_enable_mem_preload_state_table` is true.
318 #[serde(default)]
319 pub mem_preload_state_table_ids_blacklist: Vec<u32>,
320
321 /// Eliminate unnecessary updates aggressively, even if it impacts performance. Enable this
322 /// only if it's confirmed that no-op updates are causing significant streaming amplification.
323 #[serde(default)]
324 pub aggressive_noop_update_elimination: bool,
325
326 /// The interval in seconds for the refresh scheduler to check and trigger scheduled refreshes.
327 #[serde(default = "default::developer::refresh_scheduler_interval_sec")]
328 pub refresh_scheduler_interval_sec: u64,
329
330 /// Determine which encoding will be used to encode join rows in operator cache.
331 #[serde(default)]
332 pub join_encoding_type: JoinEncodingType,
333
334 /// The timeout for reading from the buffer of the sync log store on barrier.
335 /// Every epoch we will attempt to read the full buffer of the sync log store.
336 /// If we hit the timeout, we will stop reading and continue.
337 #[serde(default = "default::developer::sync_log_store_pause_duration_ms")]
338 pub sync_log_store_pause_duration_ms: usize,
339
340 /// The max buffer size for sync logstore, before we start flushing.
341 #[serde(default = "default::developer::sync_log_store_buffer_size")]
342 pub sync_log_store_buffer_size: usize,
343
344 /// Disable the optimized dispatcher path for sync log store.
345 #[serde(default = "default::developer::disable_sync_log_store_dispatcher")]
346 pub disable_sync_log_store_dispatcher: bool,
347
348 /// Cache policy for partition cache in streaming over window.
349 /// Can be `full`, `recent`, `recent_first_n` or `recent_last_n`.
350 #[serde(default)]
351 pub over_window_cache_policy: OverWindowCachePolicy,
352
353 /// When enabled, vnode stats pruning is applied in production.
354 /// When disabled, vnode stats pruning is in dry-run mode: we still maintain vnode stats
355 /// and verify that pruning would be correct, but we don't actually use the pruning
356 /// results — we still use cache and storage to fulfill the read. This is useful for
357 /// validating the correctness of vnode stats pruning before enabling it in production.
358 #[serde(default = "default::developer::enable_state_table_vnode_stats_pruning")]
359 pub enable_state_table_vnode_stats_pruning: bool,
360
361 /// Runtime switch for the sort buffer of `LocalityProviderExecutor`. Only takes effect on
362 /// streaming jobs planned with the `enable_locality_sort_buffer` session variable enabled.
363 /// Setting it to false makes such locality providers always pass through input directly.
364 #[serde(default = "default::developer::enable_locality_sort_buffer")]
365 pub enable_locality_sort_buffer: bool,
366
367 /// The number of input rows within one epoch after which the sort buffer of
368 /// `LocalityProviderExecutor` activates. Below this threshold, input is passed through
369 /// directly without buffering, so that no extra latency is introduced for normal workloads.
370 #[serde(default = "default::developer::locality_sort_buffer_activate_threshold")]
371 pub locality_sort_buffer_activate_threshold: u64,
372
373 /// Cache refill policy for streaming cache refill feature.
374 /// Can be `enabled`, `disabled`, `streaming`, `serving` or `both`.
375 #[serde(default = "default::developer::cache_refill_policy")]
376 pub cache_refill_policy: CacheRefillPolicy,
377
378 /// Whether `MaterializeExecutor` enables vnode key stats for its state table.
379 #[serde(default = "default::developer::enable_vnode_key_stats_for_materialize")]
380 pub enable_vnode_key_stats_for_materialize: bool,
381
382 /// The maximum number of kv log store readers that can concurrently read historical data
383 /// (i.e., from the state store) during initialization. A reader is considered "initializing"
384 /// until it has read at least one row from the historical stream or the stream returns empty.
385 /// Set to 0 to disable the limit (unlimited concurrency).
386 #[serde(default = "default::developer::max_concurrent_kv_log_store_historical_read")]
387 pub max_concurrent_kv_log_store_historical_read: usize,
388
389 #[serde(default, flatten)]
390 #[serde_prefix_all(skip)]
391 #[config_doc(omitted)]
392 pub unrecognized: Unrecognized<Self>,
393}
394
395impl StreamingDeveloperConfig {
396 pub fn snapshot_iter_rebuild_interval(&self) -> Duration {
397 let rebuild_interval = if self.snapshot_iter_rebuild_interval_secs < 10 {
398 tracing::warn!(
399 "too small rebuild_interval {} second. rewrite to 10",
400 self.snapshot_iter_rebuild_interval_secs
401 );
402 10
403 } else {
404 self.snapshot_iter_rebuild_interval_secs
405 };
406 Duration::from_secs(rebuild_interval)
407 }
408}
409
410impl StreamingConfig {
411 /// Returns the dot-separated keys of all unrecognized fields, including those in `developer` section.
412 pub fn unrecognized_keys(&self) -> impl Iterator<Item = String> {
413 std::iter::from_coroutine(
414 #[coroutine]
415 || {
416 for k in self.unrecognized.inner().keys() {
417 yield format!("streaming.{k}");
418 }
419 for k in self.developer.unrecognized.inner().keys() {
420 yield format!("streaming.developer.{k}");
421 }
422 },
423 )
424 }
425}
426
427pub mod default {
428 pub use crate::config::default::developer;
429
430 pub mod streaming {
431 use tracing::info;
432
433 use crate::config::AsyncStackTraceOption;
434 use crate::util::env_var::env_var_is_true;
435
436 pub fn in_flight_barrier_nums() -> usize {
437 // quick fix
438 // TODO: remove this limitation from code
439 10000
440 }
441
442 pub fn snapshot_backfill_finish_max_lagged_barriers() -> usize {
443 100
444 }
445
446 pub fn snapshot_backfill_barrier_amplification_factor() -> usize {
447 1
448 }
449
450 pub fn async_stack_trace() -> AsyncStackTraceOption {
451 AsyncStackTraceOption::default()
452 }
453
454 pub fn unique_user_stream_errors() -> usize {
455 10
456 }
457
458 pub fn unsafe_disable_strict_consistency() -> bool {
459 false
460 }
461
462 pub fn default_enable_mem_preload_state_table() -> bool {
463 if env_var_is_true("DEFAULT_ENABLE_MEM_PRELOAD_STATE_TABLE") {
464 info!("enabled mem_preload_state_table globally by env var");
465 true
466 } else {
467 false
468 }
469 }
470 }
471}