Skip to main content

risingwave_meta_node/
lib.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(coverage, feature(coverage_attribute))]
16
17mod server;
18
19use std::path::PathBuf;
20use std::time::Duration;
21
22use clap::Parser;
23use educe::Educe;
24pub use error::{MetaError, MetaResult};
25use redact::Secret;
26use risingwave_common::config::OverrideConfig;
27use risingwave_common::license::LicenseKey;
28use risingwave_common::system_param::StateStoreUrl;
29use risingwave_common::util::meta_addr::MetaAddressStrategy;
30use risingwave_common::util::resource_util;
31use risingwave_common::util::tokio_util::sync::CancellationToken;
32use risingwave_common::{GIT_SHA, RW_VERSION};
33use risingwave_common_heap_profiling::HeapProfiler;
34use risingwave_meta::barrier::snapshot_backfill_max_pending_barrier_num;
35use risingwave_meta::*;
36use risingwave_meta_service::*;
37use risingwave_sqlparser::ast::SqlOptionRedaction;
38pub use rpc::{ElectionClient, ElectionMember};
39use server::rpc_serve;
40pub use server::started::get as is_server_started;
41
42use crate::manager::MetaOpts;
43
44#[derive(Educe, Clone, Parser, OverrideConfig)]
45#[educe(Debug)]
46#[command(version, about = "The central metadata management service")]
47pub struct MetaNodeOpts {
48    // TODO: use `SocketAddr`
49    #[clap(long, env = "RW_LISTEN_ADDR", default_value = "127.0.0.1:5690")]
50    pub listen_addr: String,
51
52    /// The address for contacting this instance of the service.
53    /// This would be synonymous with the service's "public address"
54    /// or "identifying address".
55    /// It will serve as a unique identifier in cluster
56    /// membership and leader election. Must be specified for meta backend.
57    #[clap(long, env = "RW_ADVERTISE_ADDR", default_value = "127.0.0.1:5690")]
58    pub advertise_addr: String,
59
60    #[clap(long, env = "RW_DASHBOARD_HOST")]
61    pub dashboard_host: Option<String>,
62
63    /// We will start a http server at this address via `MetricsManager`.
64    /// Then the prometheus instance will poll the metrics from this address.
65    #[clap(long, env = "RW_PROMETHEUS_HOST", alias = "prometheus-host")]
66    pub prometheus_listener_addr: Option<String>,
67
68    /// Endpoint of the SQL service, make it non-option when SQL service is required.
69    #[clap(long, hide = true, env = "RW_SQL_ENDPOINT")]
70    pub sql_endpoint: Option<Secret<String>>,
71
72    /// Username of sql backend, required when meta backend set to MySQL or PostgreSQL.
73    #[clap(long, hide = true, env = "RW_SQL_USERNAME", default_value = "")]
74    pub sql_username: String,
75
76    /// Password of sql backend, required when meta backend set to MySQL or PostgreSQL.
77    #[clap(long, hide = true, env = "RW_SQL_PASSWORD", default_value = "")]
78    pub sql_password: Secret<String>,
79
80    /// Database of sql backend, required when meta backend set to MySQL or PostgreSQL.
81    #[clap(long, hide = true, env = "RW_SQL_DATABASE", default_value = "")]
82    pub sql_database: String,
83
84    /// Params for the URL connection, such as `sslmode=disable`.
85    /// Example: `param1=value1&param2=value2`
86    #[clap(long, hide = true, env = "RW_SQL_URL_PARAMS")]
87    pub sql_url_params: Option<String>,
88
89    /// The HTTP REST-API address of the Prometheus instance associated to this cluster.
90    /// This address is used to serve `PromQL` queries to Prometheus.
91    /// It is also used by Grafana Dashboard Service to fetch metrics and visualize them.
92    #[clap(long, env = "RW_PROMETHEUS_ENDPOINT")]
93    pub prometheus_endpoint: Option<String>,
94
95    /// The additional selector used when querying Prometheus.
96    ///
97    /// The format is same as `PromQL`. Example: `instance="foo",namespace="bar"`
98    #[clap(long, env = "RW_PROMETHEUS_SELECTOR")]
99    pub prometheus_selector: Option<String>,
100
101    /// Default tag for the endpoint created when creating a privatelink connection.
102    /// Will be appended to the tags specified in the `tags` field in with clause in `create
103    /// connection`.
104    #[clap(long, hide = true, env = "RW_PRIVATELINK_ENDPOINT_DEFAULT_TAGS")]
105    pub privatelink_endpoint_default_tags: Option<String>,
106
107    #[clap(long, hide = true, env = "RW_VPC_ID")]
108    pub vpc_id: Option<String>,
109
110    #[clap(long, hide = true, env = "RW_VPC_SECURITY_GROUP_ID")]
111    pub security_group_id: Option<String>,
112
113    /// The path of `risingwave.toml` configuration file.
114    ///
115    /// If empty, default configuration values will be used.
116    #[clap(long, env = "RW_CONFIG_PATH", default_value = "")]
117    pub config_path: String,
118
119    #[clap(long, hide = true, env = "RW_BACKEND", value_enum)]
120    #[override_opts(path = meta.backend)]
121    pub backend: Option<MetaBackend>,
122
123    /// The interval of periodic barrier.
124    #[clap(long, hide = true, env = "RW_BARRIER_INTERVAL_MS")]
125    #[override_opts(path = system.barrier_interval_ms)]
126    pub barrier_interval_ms: Option<u32>,
127
128    /// Target size of the Sstable.
129    #[clap(long, hide = true, env = "RW_SSTABLE_SIZE_MB")]
130    #[override_opts(path = system.sstable_size_mb)]
131    pub sstable_size_mb: Option<u32>,
132
133    /// Size of each block in bytes in SST.
134    #[clap(long, hide = true, env = "RW_BLOCK_SIZE_KB")]
135    #[override_opts(path = system.block_size_kb)]
136    pub block_size_kb: Option<u32>,
137
138    /// Deprecated: Bloom filter is no longer a supported SST filter implementation.
139    #[clap(long, hide = true, env = "RW_BLOOM_FALSE_POSITIVE")]
140    #[override_opts(path = system.bloom_false_positive)]
141    pub bloom_false_positive: Option<f64>,
142
143    /// State store url
144    #[clap(long, hide = true, env = "RW_STATE_STORE")]
145    #[override_opts(path = system.state_store)]
146    pub state_store: Option<StateStoreUrl>,
147
148    /// Remote directory for storing data and metadata objects.
149    #[clap(long, hide = true, env = "RW_DATA_DIRECTORY")]
150    #[override_opts(path = system.data_directory)]
151    pub data_directory: Option<String>,
152
153    /// Whether config object storage bucket lifecycle to purge stale data.
154    #[clap(long, hide = true, env = "RW_DO_NOT_CONFIG_BUCKET_LIFECYCLE")]
155    #[override_opts(path = meta.do_not_config_object_storage_lifecycle)]
156    pub do_not_config_object_storage_lifecycle: Option<bool>,
157
158    /// Remote storage url for storing snapshots.
159    #[clap(long, hide = true, env = "RW_BACKUP_STORAGE_URL")]
160    #[override_opts(path = system.backup_storage_url)]
161    pub backup_storage_url: Option<String>,
162
163    /// Remote directory for storing snapshots.
164    #[clap(long, hide = true, env = "RW_BACKUP_STORAGE_DIRECTORY")]
165    #[override_opts(path = system.backup_storage_directory)]
166    pub backup_storage_directory: Option<String>,
167
168    /// Enable heap profile dump when memory usage is high.
169    #[clap(long, hide = true, env = "RW_HEAP_PROFILING_DIR")]
170    #[override_opts(path = server.heap_profiling.dir)]
171    pub heap_profiling_dir: Option<String>,
172
173    /// Exit if idle for a certain period of time.
174    #[clap(long, hide = true, env = "RW_DANGEROUS_MAX_IDLE_SECS")]
175    #[override_opts(path = meta.dangerous_max_idle_secs)]
176    pub dangerous_max_idle_secs: Option<u64>,
177
178    /// Endpoint of the connector node.
179    #[deprecated = "connector node has been deprecated."]
180    #[clap(long, hide = true, env = "RW_CONNECTOR_RPC_ENDPOINT")]
181    pub connector_rpc_endpoint: Option<String>,
182
183    /// The license key to activate enterprise features.
184    #[clap(long, hide = true, env = "RW_LICENSE_KEY")]
185    #[override_opts(path = system.license_key)]
186    pub license_key: Option<LicenseKey>,
187
188    /// The path of the license key file to be watched and hot-reloaded.
189    #[clap(long, env = "RW_LICENSE_KEY_PATH")]
190    pub license_key_path: Option<PathBuf>,
191
192    /// 128-bit AES key for secret store in HEX format.
193    #[clap(long, hide = true, env = "RW_SECRET_STORE_PRIVATE_KEY_HEX")]
194    pub secret_store_private_key_hex: Option<Secret<String>>,
195
196    /// The path of the temp secret file directory.
197    #[clap(
198        long,
199        hide = true,
200        env = "RW_TEMP_SECRET_FILE_DIR",
201        default_value = "./secrets"
202    )]
203    pub temp_secret_file_dir: String,
204
205    /// Address of the serverless backfill controller.
206    /// Needed if meta receives a streaming job with serverless backfill enabled.
207    /// Feature disabled by default.
208    #[clap(long, env = "RW_SBC_ADDR", default_value = "")]
209    pub serverless_backfill_controller_addr: String,
210}
211
212impl risingwave_common::opts::Opts for MetaNodeOpts {
213    fn name() -> &'static str {
214        "meta"
215    }
216
217    fn meta_addr(&self) -> MetaAddressStrategy {
218        format!("http://{}", self.listen_addr)
219            .parse()
220            .expect("invalid listen address")
221    }
222}
223
224use std::future::Future;
225use std::pin::Pin;
226use std::sync::Arc;
227
228use risingwave_common::config::{MetaBackend, RwConfig, load_config};
229use tracing::info;
230
231/// Start meta node
232pub fn start(
233    opts: MetaNodeOpts,
234    shutdown: CancellationToken,
235) -> Pin<Box<dyn Future<Output = ()> + Send>> {
236    // WARNING: don't change the function signature. Making it `async fn` will cause
237    // slow compile in release mode.
238    Box::pin(async move {
239        info!("Starting meta node");
240        info!("> options: {:?}", opts);
241        let config = load_config(&opts.config_path, &opts);
242        info!("> config: {:?}", config);
243        info!("> version: {} ({})", RW_VERSION, GIT_SHA);
244        let listen_addr = opts.listen_addr.parse().unwrap();
245        let dashboard_addr = opts.dashboard_host.map(|x| x.parse().unwrap());
246        let prometheus_addr = opts.prometheus_listener_addr.map(|x| x.parse().unwrap());
247        let meta_store_config = config.meta.meta_store_config.clone();
248        let backend = match config.meta.backend {
249            MetaBackend::Mem => {
250                if opts.sql_endpoint.is_some() {
251                    tracing::warn!("`--sql-endpoint` is ignored when using `mem` backend");
252                }
253                MetaStoreBackend::Mem
254            }
255            MetaBackend::Sql => MetaStoreBackend::Sql {
256                endpoint: opts
257                    .sql_endpoint
258                    .expect("sql endpoint is required")
259                    .expose_secret()
260                    .clone(),
261                config: meta_store_config,
262            },
263            MetaBackend::Sqlite => MetaStoreBackend::Sql {
264                endpoint: format!(
265                    "sqlite://{}?mode=rwc",
266                    opts.sql_endpoint
267                        .expect("sql endpoint is required")
268                        .expose_secret()
269                ),
270                config: meta_store_config,
271            },
272            MetaBackend::Postgres => MetaStoreBackend::Sql {
273                endpoint: format!(
274                    "postgres://{}:{}@{}/{}{}",
275                    opts.sql_username,
276                    opts.sql_password.expose_secret(),
277                    opts.sql_endpoint
278                        .expect("sql endpoint is required")
279                        .expose_secret(),
280                    opts.sql_database,
281                    if let Some(params) = &opts.sql_url_params
282                        && !params.is_empty()
283                    {
284                        format!("?{}", params)
285                    } else {
286                        "".to_owned()
287                    }
288                ),
289                config: meta_store_config,
290            },
291            MetaBackend::Mysql => MetaStoreBackend::Sql {
292                endpoint: format!(
293                    "mysql://{}:{}@{}/{}{}",
294                    opts.sql_username,
295                    opts.sql_password.expose_secret(),
296                    opts.sql_endpoint
297                        .expect("sql endpoint is required")
298                        .expose_secret(),
299                    opts.sql_database,
300                    if let Some(params) = &opts.sql_url_params
301                        && !params.is_empty()
302                    {
303                        format!("?{}", params)
304                    } else {
305                        "".to_owned()
306                    }
307                ),
308                config: meta_store_config,
309            },
310        };
311        validate_config(&config);
312
313        let total_memory_bytes = resource_util::memory::system_memory_available_bytes();
314        let heap_profiler =
315            HeapProfiler::new(total_memory_bytes, config.server.heap_profiling.clone());
316        // Run a background heap profiler
317        heap_profiler.start();
318
319        let secret_store_private_key = opts
320            .secret_store_private_key_hex
321            .map(|key| hex::decode(key.expose_secret()).unwrap());
322        let max_heartbeat_interval =
323            Duration::from_secs(config.meta.max_heartbeat_interval_secs as u64);
324        let max_idle_ms = config.meta.dangerous_max_idle_secs.unwrap_or(0) * 1000;
325        let in_flight_barrier_nums = config.streaming.in_flight_barrier_nums;
326        let snapshot_backfill_finish_max_lagged_barriers = config
327            .streaming
328            .snapshot_backfill_finish_max_lagged_barriers;
329        let snapshot_backfill_barrier_amplification_factor = config
330            .streaming
331            .snapshot_backfill_barrier_amplification_factor;
332        let privatelink_endpoint_default_tags =
333            opts.privatelink_endpoint_default_tags.map(|tags| {
334                tags.split(',')
335                    .map(|s| {
336                        let key_val = s.split_once('=').unwrap();
337                        (key_val.0.to_owned(), key_val.1.to_owned())
338                    })
339                    .collect()
340            });
341
342        let add_info = AddressInfo {
343            advertise_addr: opts.advertise_addr.clone(),
344            listen_addr,
345            prometheus_addr,
346            dashboard_addr,
347        };
348
349        const MIN_TIMEOUT_INTERVAL_SEC: u64 = 20;
350        let compaction_task_max_progress_interval_secs = {
351            let retry_config = &config.storage.object_store.retry;
352            let max_streaming_read_timeout_ms = (retry_config.streaming_read_attempt_timeout_ms
353                + retry_config.req_backoff_max_delay_ms)
354                * retry_config.streaming_read_retry_attempts as u64;
355            let max_streaming_upload_timeout_ms = (retry_config
356                .streaming_upload_attempt_timeout_ms
357                + retry_config.req_backoff_max_delay_ms)
358                * retry_config.streaming_upload_retry_attempts as u64;
359            let max_upload_timeout_ms = (retry_config.upload_attempt_timeout_ms
360                + retry_config.req_backoff_max_delay_ms)
361                * retry_config.upload_retry_attempts as u64;
362            let max_read_timeout_ms = (retry_config.read_attempt_timeout_ms
363                + retry_config.req_backoff_max_delay_ms)
364                * retry_config.read_retry_attempts as u64;
365            let max_timeout_ms = max_streaming_read_timeout_ms
366                .max(max_upload_timeout_ms)
367                .max(max_streaming_upload_timeout_ms)
368                .max(max_read_timeout_ms)
369                .max(config.meta.compaction_task_max_progress_interval_secs * 1000);
370            max_timeout_ms / 1000
371        } + MIN_TIMEOUT_INTERVAL_SEC;
372
373        Box::pin(rpc_serve(
374            add_info,
375            backend,
376            max_heartbeat_interval,
377            config.meta.meta_leader_lease_secs,
378            config.server.clone(),
379            MetaOpts {
380                enable_recovery: !config.meta.disable_recovery,
381                clean_all_foreground_jobs_on_recovery: config
382                    .meta
383                    .clean_all_foreground_jobs_on_recovery,
384                disable_automatic_parallelism_control: config
385                    .meta
386                    .disable_automatic_parallelism_control,
387                parallelism_control_batch_size: config.meta.parallelism_control_batch_size,
388                parallelism_control_trigger_period_sec: config
389                    .meta
390                    .parallelism_control_trigger_period_sec,
391                parallelism_control_trigger_first_delay_sec: config
392                    .meta
393                    .parallelism_control_trigger_first_delay_sec,
394                in_flight_barrier_nums,
395                snapshot_backfill_finish_max_lagged_barriers,
396                snapshot_backfill_barrier_amplification_factor,
397                max_idle_ms,
398                compaction_deterministic_test: config.meta.enable_compaction_deterministic,
399                default_parallelism: config.meta.default_parallelism,
400                vacuum_interval_sec: config.meta.vacuum_interval_sec,
401                time_travel_vacuum_interval_sec: config
402                    .meta
403                    .developer
404                    .time_travel_vacuum_interval_sec,
405                time_travel_vacuum_max_version_count: config
406                    .meta
407                    .developer
408                    .time_travel_vacuum_max_version_count,
409                vacuum_spin_interval_ms: config.meta.vacuum_spin_interval_ms,
410                iceberg_gc_interval_sec: config.meta.iceberg_gc_interval_sec,
411                iceberg_orphan_file_cleanup_interval_sec: config
412                    .meta
413                    .iceberg_orphan_file_cleanup_interval_sec,
414                iceberg_compaction_report_timeout_sec: config
415                    .meta
416                    .iceberg_compaction_report_timeout_sec,
417                iceberg_compaction_config_refresh_interval_sec: config
418                    .meta
419                    .iceberg_compaction_config_refresh_interval_sec,
420                iceberg_compaction_schedule_on_startup: config
421                    .meta
422                    .iceberg_compaction_schedule_on_startup,
423                hummock_version_checkpoint_interval_sec: config
424                    .meta
425                    .hummock_version_checkpoint_interval_sec,
426                enable_hummock_data_archive: config.meta.enable_hummock_data_archive,
427                checkpoint_compression_algorithm: config.meta.checkpoint_compression_algorithm,
428                checkpoint_read_chunk_size: config.meta.checkpoint_read_chunk_size,
429                checkpoint_read_max_in_flight_chunks: config
430                    .meta
431                    .checkpoint_read_max_in_flight_chunks,
432                hummock_time_travel_snapshot_interval: config
433                    .meta
434                    .hummock_time_travel_snapshot_interval,
435                hummock_time_travel_sst_info_fetch_batch_size: config
436                    .meta
437                    .developer
438                    .hummock_time_travel_sst_info_fetch_batch_size,
439                hummock_time_travel_sst_info_insert_batch_size: config
440                    .meta
441                    .developer
442                    .hummock_time_travel_sst_info_insert_batch_size,
443                hummock_time_travel_epoch_version_insert_batch_size: config
444                    .meta
445                    .developer
446                    .hummock_time_travel_epoch_version_insert_batch_size,
447                hummock_time_travel_delta_fetch_batch_size: config
448                    .meta
449                    .developer
450                    .hummock_time_travel_delta_fetch_batch_size,
451                hummock_gc_history_insert_batch_size: config
452                    .meta
453                    .developer
454                    .hummock_gc_history_insert_batch_size,
455                hummock_time_travel_filter_out_objects_batch_size: config
456                    .meta
457                    .developer
458                    .hummock_time_travel_filter_out_objects_batch_size,
459                hummock_time_travel_filter_out_objects_v1: config
460                    .meta
461                    .developer
462                    .hummock_time_travel_filter_out_objects_v1,
463                hummock_time_travel_filter_out_objects_list_version_batch_size: config
464                    .meta
465                    .developer
466                    .hummock_time_travel_filter_out_objects_list_version_batch_size,
467                hummock_time_travel_filter_out_objects_list_delta_batch_size: config
468                    .meta
469                    .developer
470                    .hummock_time_travel_filter_out_objects_list_delta_batch_size,
471                min_delta_log_num_for_hummock_version_checkpoint: config
472                    .meta
473                    .min_delta_log_num_for_hummock_version_checkpoint,
474                min_sst_retention_time_sec: config.meta.min_sst_retention_time_sec,
475                full_gc_interval_sec: config.meta.full_gc_interval_sec,
476                full_gc_object_limit: config.meta.full_gc_object_limit,
477                gc_history_retention_time_sec: config.meta.gc_history_retention_time_sec,
478                max_inflight_time_travel_query: config.meta.max_inflight_time_travel_query,
479                enable_committed_sst_sanity_check: config.meta.enable_committed_sst_sanity_check,
480                periodic_compaction_interval_sec: config.meta.periodic_compaction_interval_sec,
481                node_num_monitor_interval_sec: config.meta.node_num_monitor_interval_sec,
482                prometheus_endpoint: opts.prometheus_endpoint,
483                prometheus_selector: opts.prometheus_selector,
484                vpc_id: opts.vpc_id,
485                security_group_id: opts.security_group_id,
486                privatelink_endpoint_default_tags,
487                periodic_space_reclaim_compaction_interval_sec: config
488                    .meta
489                    .periodic_space_reclaim_compaction_interval_sec,
490                telemetry_enabled: config.server.telemetry_enabled,
491                periodic_ttl_reclaim_compaction_interval_sec: config
492                    .meta
493                    .periodic_ttl_reclaim_compaction_interval_sec,
494                periodic_tombstone_reclaim_compaction_interval_sec: config
495                    .meta
496                    .periodic_tombstone_reclaim_compaction_interval_sec,
497                periodic_scheduling_compaction_group_split_interval_sec: config
498                    .meta
499                    .periodic_scheduling_compaction_group_split_interval_sec,
500                enable_compaction_group_normalize: config.meta.enable_compaction_group_normalize,
501                max_normalize_splits_per_round: config.meta.max_normalize_splits_per_round,
502                periodic_scheduling_compaction_group_merge_interval_sec: config
503                    .meta
504                    .periodic_scheduling_compaction_group_merge_interval_sec,
505                compaction_group_merge_dimension_threshold: config
506                    .meta
507                    .compaction_group_merge_dimension_threshold,
508                table_high_write_throughput_threshold: config
509                    .meta
510                    .table_high_write_throughput_threshold,
511                table_low_write_throughput_threshold: config
512                    .meta
513                    .table_low_write_throughput_threshold,
514                partition_vnode_count: config.meta.partition_vnode_count,
515                compact_task_table_size_partition_threshold_low: config
516                    .meta
517                    .compact_task_table_size_partition_threshold_low,
518                compact_task_table_size_partition_threshold_high: config
519                    .meta
520                    .compact_task_table_size_partition_threshold_high,
521                do_not_config_object_storage_lifecycle: config
522                    .meta
523                    .do_not_config_object_storage_lifecycle,
524                compaction_task_max_heartbeat_interval_secs: config
525                    .meta
526                    .compaction_task_max_heartbeat_interval_secs,
527                compaction_task_max_progress_interval_secs,
528                compaction_task_id_refill_capacity: config.meta.compaction_task_id_refill_capacity,
529                compaction_config: Some(config.meta.compaction_config),
530                hybrid_partition_node_count: config.meta.hybrid_partition_vnode_count,
531                event_log_enabled: config.meta.event_log_enabled,
532                event_log_channel_max_size: config.meta.event_log_channel_max_size,
533                advertise_addr: opts.advertise_addr,
534                cached_traces_num: config.meta.developer.cached_traces_num,
535                cached_traces_memory_limit_bytes: config
536                    .meta
537                    .developer
538                    .cached_traces_memory_limit_bytes,
539                enable_trivial_move: config.meta.developer.enable_trivial_move,
540                enable_check_task_level_overlap: config
541                    .meta
542                    .developer
543                    .enable_check_task_level_overlap,
544                enable_dropped_column_reclaim: config.meta.enable_dropped_column_reclaim,
545                split_group_size_ratio: config.meta.split_group_size_ratio,
546                refresh_scheduler_interval_sec: config
547                    .streaming
548                    .developer
549                    .refresh_scheduler_interval_sec,
550                table_write_throughput_retention_seconds: config
551                    .meta
552                    .table_write_throughput_retention_seconds,
553                object_store_config: config.storage.object_store,
554                max_trivial_move_task_count_per_loop: config
555                    .meta
556                    .developer
557                    .max_trivial_move_task_count_per_loop,
558                max_get_task_probe_times: config.meta.developer.max_get_task_probe_times,
559                secret_store_private_key,
560                temp_secret_file_dir: opts.temp_secret_file_dir,
561                actor_cnt_per_worker_parallelism_hard_limit: config
562                    .meta
563                    .developer
564                    .actor_cnt_per_worker_parallelism_hard_limit,
565                actor_cnt_per_worker_parallelism_soft_limit: config
566                    .meta
567                    .developer
568                    .actor_cnt_per_worker_parallelism_soft_limit,
569                table_change_log_insert_batch_size: config
570                    .meta
571                    .developer
572                    .table_change_log_insert_batch_size,
573                table_change_log_delete_batch_size: config
574                    .meta
575                    .developer
576                    .table_change_log_delete_batch_size,
577                table_change_log_truncate_interval_sec: config
578                    .meta
579                    .developer
580                    .table_change_log_truncate_interval_sec,
581                license_key_path: opts.license_key_path,
582                compute_client_config: config.meta.developer.compute_client_config.clone(),
583                stream_client_config: config.meta.developer.stream_client_config.clone(),
584                frontend_client_config: config.meta.developer.frontend_client_config.clone(),
585                sql_option_redaction: Arc::new(SqlOptionRedaction::new(
586                    config.batch.redact_sql_option_keywords,
587                    config.batch.redact_sql_option_allowlist,
588                )),
589                cdc_table_split_init_sleep_interval_splits: config
590                    .meta
591                    .cdc_table_split_init_sleep_interval_splits,
592                cdc_table_split_init_sleep_duration_millis: config
593                    .meta
594                    .cdc_table_split_init_sleep_duration_millis,
595                cdc_table_split_init_insert_batch_size: config
596                    .meta
597                    .cdc_table_split_init_insert_batch_size,
598
599                enable_legacy_table_migration: config.meta.enable_legacy_table_migration,
600                pause_on_next_bootstrap_offline: config.meta.pause_on_next_bootstrap_offline,
601                serverless_backfill_controller_addr: opts.serverless_backfill_controller_addr,
602            },
603            config.system.into_init_system_params(),
604            config.session_init,
605            shutdown,
606        ))
607        .await
608        .unwrap();
609    })
610}
611
612fn validate_config(config: &RwConfig) {
613    if config.meta.meta_leader_lease_secs <= 2 {
614        let error_msg = "`meta_leader_lease_secs` must be greater than 2";
615        tracing::error!(error_msg);
616        panic!("{}", error_msg);
617    }
618
619    if config.meta.parallelism_control_batch_size == 0 {
620        let error_msg = "`parallelism_control_batch_size` must be greater than 0";
621        tracing::error!(error_msg);
622        panic!("{}", error_msg);
623    }
624
625    if config.meta.checkpoint_read_chunk_size == 0 {
626        let error_msg = "`checkpoint_read_chunk_size` must be greater than 0";
627        tracing::error!(error_msg);
628        panic!("{}", error_msg);
629    }
630
631    if config.meta.checkpoint_read_max_in_flight_chunks == 0 {
632        let error_msg = "`checkpoint_read_max_in_flight_chunks` must be greater than 0";
633        tracing::error!(error_msg);
634        panic!("{}", error_msg);
635    }
636
637    if config.meta.compaction_task_id_refill_capacity == 0 {
638        let error_msg = "`compaction_task_id_refill_capacity` must be greater than 0";
639        tracing::error!(error_msg);
640        panic!("{}", error_msg);
641    }
642
643    if config.meta.iceberg_compaction_report_timeout_sec == 0 {
644        let error_msg = "`iceberg_compaction_report_timeout_sec` must be greater than 0";
645        tracing::error!(error_msg);
646        panic!("{}", error_msg);
647    }
648
649    if config.meta.iceberg_compaction_config_refresh_interval_sec == 0 {
650        let error_msg = "`iceberg_compaction_config_refresh_interval_sec` must be greater than 0";
651        tracing::error!(error_msg);
652        panic!("{}", error_msg);
653    }
654
655    // A snapshot backfill partial graph drains its queued catch-up barriers only when a new
656    // upstream barrier arrives, and that barrier is queued first, so the queue shrinks by at
657    // most `limit - 1` per upstream barrier. A limit of 1 could never empty the queue.
658    if snapshot_backfill_max_pending_barrier_num(
659        config.streaming.in_flight_barrier_nums,
660        config
661            .streaming
662            .snapshot_backfill_barrier_amplification_factor,
663    ) < 2
664    {
665        let error_msg = "`in_flight_barrier_nums` times `snapshot_backfill_barrier_amplification_factor` must be at least 2";
666        tracing::error!(error_msg);
667        panic!("{}", error_msg);
668    }
669}