1use std::ops::Deref;
16use std::path::PathBuf;
17use std::sync::Arc;
18use std::sync::atomic::AtomicU32;
19
20use anyhow::Context;
21use risingwave_common::config::{
22 CompactionConfig, DefaultParallelism, ObjectStoreConfig, RpcClientConfig,
23};
24use risingwave_common::session_config::SessionConfig;
25use risingwave_common::system_param::reader::SystemParamsReader;
26use risingwave_common::{bail, system_param};
27use risingwave_meta_model::prelude::Cluster;
28use risingwave_pb::meta::SystemParams;
29use risingwave_rpc_client::{
30 FrontendClientPool, FrontendClientPoolRef, StreamClientPool, StreamClientPoolRef,
31};
32use risingwave_sqlparser::ast::RedactSqlOptionKeywordsRef;
33use sea_orm::EntityTrait;
34
35use crate::MetaResult;
36use crate::barrier::SharedActorInfos;
37use crate::controller::SqlMetaStore;
38use crate::controller::id::{
39 IdGeneratorManager as SqlIdGeneratorManager, IdGeneratorManagerRef as SqlIdGeneratorManagerRef,
40};
41use crate::controller::session_params::{SessionParamsController, SessionParamsControllerRef};
42use crate::controller::system_param::{SystemParamsController, SystemParamsControllerRef};
43use crate::hummock::sequence::SequenceGenerator;
44use crate::manager::event_log::{EventLogManagerRef, start_event_log_manager};
45use crate::manager::{IdleManager, IdleManagerRef, NotificationManager, NotificationManagerRef};
46use crate::model::ClusterId;
47
48#[derive(Clone)]
51pub struct MetaSrvEnv {
52 id_gen_manager_impl: SqlIdGeneratorManagerRef,
54
55 system_param_manager_impl: SystemParamsControllerRef,
57
58 session_param_manager_impl: SessionParamsControllerRef,
60
61 meta_store_impl: SqlMetaStore,
63
64 notification_manager: NotificationManagerRef,
66
67 pub shared_actor_info: SharedActorInfos,
68
69 stream_client_pool: StreamClientPoolRef,
71
72 frontend_client_pool: FrontendClientPoolRef,
74
75 idle_manager: IdleManagerRef,
77
78 event_log_manager: EventLogManagerRef,
79
80 cluster_id: ClusterId,
82
83 pub hummock_seq: Arc<SequenceGenerator>,
84
85 await_tree_reg: await_tree::Registry,
87
88 pub opts: Arc<MetaOpts>,
90
91 actor_id_generator: Arc<AtomicU32>,
92}
93
94#[derive(Clone, serde::Serialize)]
96pub struct MetaOpts {
97 pub enable_recovery: bool,
100 pub disable_automatic_parallelism_control: bool,
102 pub parallelism_control_batch_size: usize,
104 pub parallelism_control_trigger_period_sec: u64,
106 pub parallelism_control_trigger_first_delay_sec: u64,
108 pub in_flight_barrier_nums: usize,
110 pub max_idle_ms: u64,
113 pub compaction_deterministic_test: bool,
115 pub default_parallelism: DefaultParallelism,
117
118 pub vacuum_interval_sec: u64,
121 pub vacuum_spin_interval_ms: u64,
124 pub iceberg_gc_interval_sec: u64,
126 pub time_travel_vacuum_interval_sec: u64,
127 pub time_travel_vacuum_max_version_count: Option<u32>,
128 pub hummock_version_checkpoint_interval_sec: u64,
130 pub enable_hummock_data_archive: bool,
131 pub checkpoint_compression_algorithm: risingwave_common::config::CheckpointCompression,
133 pub checkpoint_read_chunk_size: usize,
135 pub checkpoint_read_max_in_flight_chunks: usize,
137 pub hummock_time_travel_snapshot_interval: u64,
138 pub hummock_time_travel_sst_info_fetch_batch_size: usize,
139 pub hummock_time_travel_sst_info_insert_batch_size: usize,
140 pub hummock_time_travel_epoch_version_insert_batch_size: usize,
141 pub hummock_time_travel_delta_fetch_batch_size: usize,
142 pub hummock_gc_history_insert_batch_size: usize,
143 pub hummock_time_travel_filter_out_objects_batch_size: usize,
144 pub hummock_time_travel_filter_out_objects_v1: bool,
145 pub hummock_time_travel_filter_out_objects_list_version_batch_size: usize,
146 pub hummock_time_travel_filter_out_objects_list_delta_batch_size: usize,
147 pub min_delta_log_num_for_hummock_version_checkpoint: u64,
152 pub min_sst_retention_time_sec: u64,
155 pub full_gc_interval_sec: u64,
157 pub full_gc_object_limit: u64,
159 pub gc_history_retention_time_sec: u64,
161 pub max_inflight_time_travel_query: u64,
163 pub enable_committed_sst_sanity_check: bool,
165 pub periodic_compaction_interval_sec: u64,
167 pub node_num_monitor_interval_sec: u64,
169 pub protect_drop_table_with_incoming_sink: bool,
171 pub prometheus_endpoint: Option<String>,
177
178 pub prometheus_selector: Option<String>,
180
181 pub vpc_id: Option<String>,
183
184 pub security_group_id: Option<String>,
186
187 pub privatelink_endpoint_default_tags: Option<Vec<(String, String)>>,
191
192 pub periodic_space_reclaim_compaction_interval_sec: u64,
194
195 pub telemetry_enabled: bool,
197 pub periodic_ttl_reclaim_compaction_interval_sec: u64,
199
200 pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
202
203 pub periodic_scheduling_compaction_group_split_interval_sec: u64,
205 pub enable_compaction_group_normalize: bool,
207 pub max_normalize_splits_per_round: u64,
209
210 pub do_not_config_object_storage_lifecycle: bool,
212
213 pub partition_vnode_count: u32,
214
215 pub table_high_write_throughput_threshold: u64,
217 pub table_low_write_throughput_threshold: u64,
219
220 pub compaction_task_max_heartbeat_interval_secs: u64,
221 pub compaction_task_max_progress_interval_secs: u64,
222 pub compaction_task_id_refill_capacity: u32,
223 pub compaction_config: Option<CompactionConfig>,
224
225 pub hybrid_partition_node_count: u32,
233
234 pub event_log_enabled: bool,
235 pub event_log_channel_max_size: u32,
236 pub advertise_addr: String,
237 pub cached_traces_num: u32,
240 pub cached_traces_memory_limit_bytes: usize,
243
244 pub enable_trivial_move: bool,
246
247 pub enable_check_task_level_overlap: bool,
249 pub enable_dropped_column_reclaim: bool,
250
251 pub split_group_size_ratio: f64,
253
254 pub refresh_scheduler_interval_sec: u64,
256
257 pub table_stat_high_write_throughput_ratio_for_split: f64,
259
260 pub table_stat_low_write_throughput_ratio_for_merge: f64,
262
263 pub table_stat_throuput_window_seconds_for_split: usize,
265
266 pub table_stat_throuput_window_seconds_for_merge: usize,
268
269 pub object_store_config: ObjectStoreConfig,
271
272 pub max_trivial_move_task_count_per_loop: usize,
274
275 pub max_get_task_probe_times: usize,
277
278 pub compact_task_table_size_partition_threshold_low: u64,
279 pub compact_task_table_size_partition_threshold_high: u64,
280
281 pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
282
283 pub compaction_group_merge_dimension_threshold: f64,
284
285 pub secret_store_private_key: Option<Vec<u8>>,
287 pub temp_secret_file_dir: String,
289
290 pub actor_cnt_per_worker_parallelism_hard_limit: usize,
292 pub actor_cnt_per_worker_parallelism_soft_limit: usize,
293
294 pub table_change_log_insert_batch_size: u64,
295 pub table_change_log_delete_batch_size: u64,
296
297 pub license_key_path: Option<PathBuf>,
298
299 pub compute_client_config: RpcClientConfig,
300 pub stream_client_config: RpcClientConfig,
301 pub frontend_client_config: RpcClientConfig,
302 pub redact_sql_option_keywords: RedactSqlOptionKeywordsRef,
303
304 pub cdc_table_split_init_sleep_interval_splits: u64,
305 pub cdc_table_split_init_sleep_duration_millis: u64,
306 pub cdc_table_split_init_insert_batch_size: u64,
307
308 pub enable_legacy_table_migration: bool,
309 pub pause_on_next_bootstrap_offline: bool,
310}
311
312impl MetaOpts {
313 pub fn test(enable_recovery: bool) -> Self {
315 Self {
316 enable_recovery,
317 disable_automatic_parallelism_control: false,
318 parallelism_control_batch_size: 1,
319 parallelism_control_trigger_period_sec: 10,
320 parallelism_control_trigger_first_delay_sec: 30,
321 in_flight_barrier_nums: 40,
322 max_idle_ms: 0,
323 compaction_deterministic_test: false,
324 default_parallelism: DefaultParallelism::Full,
325 vacuum_interval_sec: 30,
326 time_travel_vacuum_interval_sec: 30,
327 time_travel_vacuum_max_version_count: None,
328 vacuum_spin_interval_ms: 0,
329 iceberg_gc_interval_sec: 3600,
330 hummock_version_checkpoint_interval_sec: 30,
331 enable_hummock_data_archive: false,
332 checkpoint_compression_algorithm:
333 risingwave_common::config::CheckpointCompression::Zstd,
334 checkpoint_read_chunk_size: 128 * 1024 * 1024,
335 checkpoint_read_max_in_flight_chunks: 4,
336 hummock_time_travel_snapshot_interval: 0,
337 hummock_time_travel_sst_info_fetch_batch_size: 10_000,
338 hummock_time_travel_sst_info_insert_batch_size: 10,
339 hummock_time_travel_epoch_version_insert_batch_size: 1000,
340 hummock_time_travel_delta_fetch_batch_size: 1000,
341 hummock_gc_history_insert_batch_size: 1000,
342 hummock_time_travel_filter_out_objects_batch_size: 1000,
343 hummock_time_travel_filter_out_objects_v1: false,
344 hummock_time_travel_filter_out_objects_list_version_batch_size: 10,
345 hummock_time_travel_filter_out_objects_list_delta_batch_size: 1000,
346 min_delta_log_num_for_hummock_version_checkpoint: 1,
347 min_sst_retention_time_sec: 3600 * 24 * 7,
348 full_gc_interval_sec: 3600 * 24 * 7,
349 full_gc_object_limit: 100_000,
350 gc_history_retention_time_sec: 3600 * 24 * 7,
351 max_inflight_time_travel_query: 1000,
352 enable_committed_sst_sanity_check: false,
353 periodic_compaction_interval_sec: 300,
354 node_num_monitor_interval_sec: 10,
355 protect_drop_table_with_incoming_sink: false,
356 prometheus_endpoint: None,
357 prometheus_selector: None,
358 vpc_id: None,
359 security_group_id: None,
360 privatelink_endpoint_default_tags: None,
361 periodic_space_reclaim_compaction_interval_sec: 60,
362 telemetry_enabled: false,
363 periodic_ttl_reclaim_compaction_interval_sec: 60,
364 periodic_tombstone_reclaim_compaction_interval_sec: 60,
365 periodic_scheduling_compaction_group_split_interval_sec: 60,
366 enable_compaction_group_normalize: false,
367 max_normalize_splits_per_round: 4,
368 compact_task_table_size_partition_threshold_low: 128 * 1024 * 1024,
369 compact_task_table_size_partition_threshold_high: 512 * 1024 * 1024,
370 table_high_write_throughput_threshold: 128 * 1024 * 1024,
371 table_low_write_throughput_threshold: 64 * 1024 * 1024,
372 do_not_config_object_storage_lifecycle: true,
373 partition_vnode_count: 32,
374 compaction_task_max_heartbeat_interval_secs: 0,
375 compaction_task_max_progress_interval_secs: 1,
376 compaction_task_id_refill_capacity: 64,
377 compaction_config: None,
378 hybrid_partition_node_count: 4,
379 event_log_enabled: false,
380 event_log_channel_max_size: 1,
381 advertise_addr: "".to_owned(),
382 cached_traces_num: 1,
383 cached_traces_memory_limit_bytes: usize::MAX,
384 enable_trivial_move: true,
385 enable_check_task_level_overlap: true,
386 enable_dropped_column_reclaim: false,
387 object_store_config: ObjectStoreConfig::default(),
388 max_trivial_move_task_count_per_loop: 256,
389 max_get_task_probe_times: 5,
390 secret_store_private_key: Some(
391 hex::decode("0123456789abcdef0123456789abcdef").unwrap(),
392 ),
393 temp_secret_file_dir: "./secrets".to_owned(),
394 actor_cnt_per_worker_parallelism_hard_limit: usize::MAX,
395 actor_cnt_per_worker_parallelism_soft_limit: usize::MAX,
396 split_group_size_ratio: 0.9,
397 table_stat_high_write_throughput_ratio_for_split: 0.5,
398 table_stat_low_write_throughput_ratio_for_merge: 0.7,
399 table_stat_throuput_window_seconds_for_split: 60,
400 table_stat_throuput_window_seconds_for_merge: 240,
401 periodic_scheduling_compaction_group_merge_interval_sec: 60 * 10,
402 compaction_group_merge_dimension_threshold: 1.2,
403 license_key_path: None,
404 compute_client_config: RpcClientConfig::default(),
405 stream_client_config: RpcClientConfig::default(),
406 frontend_client_config: RpcClientConfig::default(),
407 redact_sql_option_keywords: Arc::new(Default::default()),
408 cdc_table_split_init_sleep_interval_splits: 1000,
409 cdc_table_split_init_sleep_duration_millis: 10,
410 cdc_table_split_init_insert_batch_size: 1000,
411 enable_legacy_table_migration: true,
412 refresh_scheduler_interval_sec: 60,
413 pause_on_next_bootstrap_offline: false,
414 table_change_log_insert_batch_size: 1000,
415 table_change_log_delete_batch_size: 1000,
416 }
417 }
418}
419
420impl MetaSrvEnv {
421 pub async fn new(
422 opts: MetaOpts,
423 mut init_system_params: SystemParams,
424 init_session_config: SessionConfig,
425 meta_store_impl: SqlMetaStore,
426 ) -> MetaResult<Self> {
427 let idle_manager = Arc::new(IdleManager::new(opts.max_idle_ms));
428 let stream_client_pool =
429 Arc::new(StreamClientPool::new(1, opts.stream_client_config.clone())); let frontend_client_pool = Arc::new(FrontendClientPool::new(
431 1,
432 opts.frontend_client_config.clone(),
433 ));
434 let event_log_manager = Arc::new(start_event_log_manager(
435 opts.event_log_enabled,
436 opts.event_log_channel_max_size,
437 ));
438
439 if opts.license_key_path.is_some()
442 && init_system_params.license_key
443 != system_param::default::license_key_opt().map(Into::into)
444 {
445 bail!(
446 "argument `--license-key-path` (or env var `RW_LICENSE_KEY_PATH`) and \
447 system parameter `license_key` (or env var `RW_LICENSE_KEY`) may not \
448 be set at the same time"
449 );
450 }
451
452 let cluster_first_launch = meta_store_impl.up().await.context(
453 "Failed to initialize the meta store, \
454 this may happen if there's existing metadata incompatible with the current version of RisingWave, \
455 e.g., downgrading from a newer release or a nightly build to an older one. \
456 For a single-node deployment, you may want to reset all data by deleting the data directory, \
457 typically located at `~/.risingwave`.",
458 )?;
459
460 let notification_manager =
461 Arc::new(NotificationManager::new(meta_store_impl.clone()).await);
462 let cluster_id = Cluster::find()
463 .one(&meta_store_impl.conn)
464 .await?
465 .map(|c| c.cluster_id.to_string().into())
466 .unwrap();
467
468 init_system_params.use_new_object_prefix_strategy = Some(cluster_first_launch);
474
475 let system_param_controller = Arc::new(
476 SystemParamsController::new(
477 meta_store_impl.clone(),
478 notification_manager.clone(),
479 init_system_params,
480 )
481 .await?,
482 );
483 let session_param_controller = Arc::new(
484 SessionParamsController::new(
485 meta_store_impl.clone(),
486 notification_manager.clone(),
487 init_session_config,
488 )
489 .await?,
490 );
491 Ok(Self {
492 id_gen_manager_impl: Arc::new(SqlIdGeneratorManager::new(&meta_store_impl.conn).await?),
493 system_param_manager_impl: system_param_controller,
494 session_param_manager_impl: session_param_controller,
495 meta_store_impl: meta_store_impl.clone(),
496 shared_actor_info: SharedActorInfos::new(notification_manager.clone()),
497 notification_manager,
498 stream_client_pool,
499 frontend_client_pool,
500 idle_manager,
501 event_log_manager,
502 cluster_id,
503 hummock_seq: Arc::new(SequenceGenerator::new(meta_store_impl.conn.clone())),
504 opts: opts.into(),
505 await_tree_reg: await_tree::Registry::new(Default::default()),
507 actor_id_generator: Arc::new(AtomicU32::new(0)),
508 })
509 }
510
511 pub fn meta_store(&self) -> SqlMetaStore {
512 self.meta_store_impl.clone()
513 }
514
515 pub fn meta_store_ref(&self) -> &SqlMetaStore {
516 &self.meta_store_impl
517 }
518
519 pub fn id_gen_manager(&self) -> &SqlIdGeneratorManagerRef {
520 &self.id_gen_manager_impl
521 }
522
523 pub fn notification_manager_ref(&self) -> NotificationManagerRef {
524 self.notification_manager.clone()
525 }
526
527 pub fn notification_manager(&self) -> &NotificationManager {
528 self.notification_manager.deref()
529 }
530
531 pub fn idle_manager_ref(&self) -> IdleManagerRef {
532 self.idle_manager.clone()
533 }
534
535 pub fn idle_manager(&self) -> &IdleManager {
536 self.idle_manager.deref()
537 }
538
539 pub fn actor_id_generator(&self) -> &AtomicU32 {
540 self.actor_id_generator.deref()
541 }
542
543 pub async fn system_params_reader(&self) -> SystemParamsReader {
544 self.system_param_manager_impl.get_params().await
545 }
546
547 pub fn system_params_manager_impl_ref(&self) -> SystemParamsControllerRef {
548 self.system_param_manager_impl.clone()
549 }
550
551 pub fn session_params_manager_impl_ref(&self) -> SessionParamsControllerRef {
552 self.session_param_manager_impl.clone()
553 }
554
555 pub fn stream_client_pool_ref(&self) -> StreamClientPoolRef {
556 self.stream_client_pool.clone()
557 }
558
559 pub fn stream_client_pool(&self) -> &StreamClientPool {
560 self.stream_client_pool.deref()
561 }
562
563 pub fn frontend_client_pool(&self) -> &FrontendClientPool {
564 self.frontend_client_pool.deref()
565 }
566
567 pub fn cluster_id(&self) -> &ClusterId {
568 &self.cluster_id
569 }
570
571 pub fn event_log_manager_ref(&self) -> EventLogManagerRef {
572 self.event_log_manager.clone()
573 }
574
575 pub fn await_tree_reg(&self) -> &await_tree::Registry {
576 &self.await_tree_reg
577 }
578
579 pub fn shared_actor_infos(&self) -> &SharedActorInfos {
580 &self.shared_actor_info
581 }
582}
583
584#[cfg(any(test, feature = "test"))]
585impl MetaSrvEnv {
586 pub async fn for_test() -> Self {
588 Self::for_test_opts(MetaOpts::test(false), |_| ()).await
589 }
590
591 pub async fn for_test_opts(
592 opts: MetaOpts,
593 on_test_system_params: impl FnOnce(&mut risingwave_pb::meta::PbSystemParams),
594 ) -> Self {
595 let mut system_params = risingwave_common::system_param::system_params_for_test();
596 on_test_system_params(&mut system_params);
597 Self::new(
598 opts,
599 system_params,
600 Default::default(),
601 SqlMetaStore::for_test().await,
602 )
603 .await
604 .unwrap()
605 }
606}