1use std::collections::HashMap;
16use std::collections::hash_map::Entry;
17use std::sync::Arc;
18
19use risingwave_common::id::JobId;
20use risingwave_common::system_param::reader::SystemParamsReader;
21use risingwave_common::util::iter_util::ZipEqDebug;
22use risingwave_meta_model::ObjectId;
23use risingwave_pb::common::{WorkerNode, WorkerType};
24use risingwave_pb::meta::object::PbObjectInfo;
25use risingwave_pb::meta::subscribe_response::{Info, Operation};
26use risingwave_pb::meta::{
27 MetaSnapshot, PbObject, PbObjectGroup, SubscribeResponse, SubscribeType,
28};
29use thiserror_ext::AsReport;
30use tokio::sync::Mutex;
31use tokio::sync::mpsc::{self, UnboundedSender};
32use tonic::Status;
33
34use crate::controller::SqlMetaStore;
35use crate::manager::WorkerKey;
36use crate::manager::notification_version::NotificationVersionGenerator;
37use crate::model::FragmentId;
38
39pub type MessageStatus = Status;
40pub type Notification = Result<SubscribeResponse, Status>;
41pub type NotificationManagerRef = Arc<NotificationManager>;
42pub type NotificationVersion = u64;
43pub const IGNORED_NOTIFICATION_VERSION: u64 = 0;
45
46pub struct FrontendNotificationBatch<'a> {
52 notification_manager: &'a NotificationManager,
53 pending: Vec<(Operation, Info)>,
54}
55
56impl<'a> FrontendNotificationBatch<'a> {
57 pub fn new(notification_manager: &'a NotificationManager) -> Self {
58 Self {
59 notification_manager,
60 pending: Vec::new(),
61 }
62 }
63
64 pub fn add(&mut self, operation: Operation, info: Info) {
65 self.pending.push((operation, info));
66 }
67
68 #[must_use]
69 pub async fn publish(self) -> Option<NotificationVersion> {
70 self.notification_manager
71 .notify_frontend_batch(self.pending)
72 .await
73 }
74}
75
76#[derive(Clone, Debug)]
77pub enum LocalNotification {
78 WorkerNodeDeleted(WorkerNode),
79 WorkerNodeActivated(WorkerNode),
80 SystemParamsChange(SystemParamsReader),
81 BatchParallelismChange,
82 ServingFragmentMappingsUpsert(Vec<FragmentId>),
83 ServingFragmentMappingsDelete(Vec<FragmentId>),
84 SourceDropped(ObjectId),
85 StreamingJobBackfillFinished(JobId),
86}
87
88#[derive(Debug)]
89enum Target {
90 Subscribers {
91 subscribe_type: SubscribeType,
92 worker_key: Option<WorkerKey>,
94 },
95 Sender(UnboundedSender<Notification>),
98}
99
100impl From<SubscribeType> for Target {
101 fn from(value: SubscribeType) -> Self {
102 Self::Subscribers {
103 subscribe_type: value,
104 worker_key: None,
105 }
106 }
107}
108
109#[derive(Debug)]
110struct Task {
111 target: Target,
112 operation: Operation,
113 info: Info,
114 version: Option<NotificationVersion>,
117}
118
119pub struct NotificationManager {
121 core: Arc<parking_lot::Mutex<NotificationManagerCore>>,
122 task_tx: UnboundedSender<Task>,
124 version_generator: Mutex<NotificationVersionGenerator>,
126}
127
128impl NotificationManager {
129 pub async fn new(meta_store_impl: SqlMetaStore) -> Self {
130 let (task_tx, mut task_rx) = mpsc::unbounded_channel::<Task>();
132 let core = Arc::new(parking_lot::Mutex::new(NotificationManagerCore::new()));
133 let core_clone = core.clone();
134 let version_generator = NotificationVersionGenerator::new(meta_store_impl)
135 .await
136 .unwrap();
137 let mut latest_version = version_generator.current_version();
138
139 tokio::spawn(async move {
140 while let Some(task) = task_rx.recv().await {
141 if let Some(version) = task.version {
142 latest_version = version;
143 }
144 let response = SubscribeResponse {
145 status: None,
146 operation: task.operation as i32,
147 info: Some(task.info),
148 version: latest_version,
149 };
150 core.lock().notify(task.target, response);
151 }
152 });
153
154 Self {
155 core: core_clone,
156 task_tx,
157 version_generator: Mutex::new(version_generator),
158 }
159 }
160
161 pub fn abort_all(&self) {
162 let mut guard = self.core.lock();
163 *guard = NotificationManagerCore::new();
164 guard.exiting = true;
165 }
166
167 #[inline(always)]
168 fn notify(
169 &self,
170 target: Target,
171 operation: Operation,
172 info: Info,
173 version: Option<NotificationVersion>,
174 ) {
175 let task = Task {
176 target,
177 operation,
178 info,
179 version,
180 };
181 self.task_tx.send(task).unwrap();
182 }
183
184 async fn notify_with_version(
186 &self,
187 target: Target,
188 operation: Operation,
189 info: Info,
190 ) -> NotificationVersion {
191 let mut version_guard = self.version_generator.lock().await;
192 version_guard.increase_version().await;
193 let version = version_guard.current_version();
194 self.notify(target, operation, info, Some(version));
195 version
196 }
197
198 async fn notify_frontend_batch(
201 &self,
202 notifications: Vec<(Operation, Info)>,
203 ) -> Option<NotificationVersion> {
204 let batch_len: NotificationVersion = notifications.len().try_into().unwrap();
205 if batch_len == 0 {
206 return None;
207 }
208
209 let mut version_guard = self.version_generator.lock().await;
210 let first_version = version_guard.current_version() + 1;
211 version_guard.increase_version_by(batch_len).await;
212 let final_version = version_guard.current_version();
213
214 for ((operation, info), version) in notifications
215 .into_iter()
216 .zip_eq_debug(first_version..=final_version)
217 {
218 self.notify(
219 SubscribeType::Frontend.into(),
220 operation,
221 info,
222 Some(version),
223 );
224 }
225
226 Some(final_version)
227 }
228
229 #[inline(always)]
231 fn notify_without_version(&self, target: Target, operation: Operation, info: Info) {
232 self.notify(target, operation, info, None);
233 }
234
235 pub fn notify_snapshot(
238 &self,
239 sender: UnboundedSender<Notification>,
240 meta_snapshot: MetaSnapshot,
241 ) {
242 self.notify_without_version(
243 Target::Sender(sender),
244 Operation::Snapshot,
245 Info::Snapshot(meta_snapshot),
246 );
247 }
248
249 pub fn notify_all_without_version(&self, operation: Operation, info: Info) {
250 for subscribe_type in [
251 SubscribeType::Frontend,
252 SubscribeType::Hummock,
253 SubscribeType::Compactor,
254 SubscribeType::Compute,
255 ] {
256 self.notify_without_version(subscribe_type.into(), operation, info.clone());
257 }
258 }
259
260 #[must_use]
261 pub async fn notify_frontend(&self, operation: Operation, info: Info) -> NotificationVersion {
262 self.notify_with_version(SubscribeType::Frontend.into(), operation, info)
263 .await
264 }
265
266 #[must_use]
267 pub async fn notify_frontend_object_info(
268 &self,
269 operation: Operation,
270 object_info: PbObjectInfo,
271 ) -> NotificationVersion {
272 self.notify_with_version(
273 SubscribeType::Frontend.into(),
274 operation,
275 Info::ObjectGroup(PbObjectGroup {
276 objects: vec![PbObject {
277 object_info: object_info.into(),
278 }],
279 dependencies: vec![],
280 }),
281 )
282 .await
283 }
284
285 pub(crate) fn notify_hummock_targeted_update(&self, worker_key: WorkerKey, info: Info) {
286 self.notify_without_version(
287 Target::Subscribers {
288 subscribe_type: SubscribeType::Hummock,
289 worker_key: Some(worker_key),
290 },
291 Operation::Update,
292 info,
293 );
294 }
295
296 pub fn notify_compute_without_version(&self, operation: Operation, info: Info) {
297 self.notify_without_version(SubscribeType::Compute.into(), operation, info)
298 }
299
300 pub fn notify_frontend_without_version(&self, operation: Operation, info: Info) {
301 self.notify_without_version(SubscribeType::Frontend.into(), operation, info)
302 }
303
304 pub fn notify_hummock(&self, operation: Operation, info: Info) {
305 self.notify_without_version(SubscribeType::Hummock.into(), operation, info)
306 }
307
308 pub fn notify_compactor(&self, operation: Operation, info: Info) {
309 self.notify_without_version(SubscribeType::Compactor.into(), operation, info)
310 }
311
312 #[cfg(any(test, feature = "test"))]
313 pub fn notify_hummock_with_version(
314 &self,
315 operation: Operation,
316 info: Info,
317 version: Option<NotificationVersion>,
318 ) {
319 self.notify(SubscribeType::Hummock.into(), operation, info, version)
320 }
321
322 pub fn notify_local_subscribers(&self, notification: LocalNotification) {
323 let mut core_guard = self.core.lock();
324 core_guard.local_senders.retain(|sender| {
325 if let Err(err) = sender.send(notification.clone()) {
326 tracing::warn!(error = %err.as_report(), "Failed to notify local subscriber");
327 return false;
328 }
329 true
330 });
331 }
332
333 pub fn delete_sender(&self, worker_type: WorkerType, worker_key: WorkerKey) {
335 let mut core_guard = self.core.lock();
336 match worker_type {
339 WorkerType::Frontend => core_guard.frontend_senders.remove(&worker_key),
340 WorkerType::ComputeNode | WorkerType::RiseCtl => {
341 core_guard.hummock_senders.remove(&worker_key)
342 }
343 WorkerType::Compactor => core_guard.compactor_senders.remove(&worker_key),
344 _ => unreachable!(),
345 };
346 }
347
348 pub fn insert_sender(
350 &self,
351 subscribe_type: SubscribeType,
352 worker_key: WorkerKey,
353 sender: UnboundedSender<Notification>,
354 ) {
355 let mut core_guard = self.core.lock();
356 if core_guard.exiting {
357 tracing::warn!("notification manager exiting.");
358 return;
359 }
360 let senders = core_guard.senders_of(subscribe_type);
361
362 senders.insert(worker_key, sender);
363 }
364
365 pub fn insert_local_sender(&self, sender: UnboundedSender<LocalNotification>) {
366 let mut core_guard = self.core.lock();
367 if core_guard.exiting {
368 tracing::warn!("notification manager exiting.");
369 return;
370 }
371 core_guard.local_senders.push(sender);
372 }
373
374 #[cfg(test)]
375 pub fn clear_local_sender(&self) {
376 self.core.lock().local_senders.clear();
377 }
378
379 pub async fn current_version(&self) -> NotificationVersion {
380 let version_guard = self.version_generator.lock().await;
381 version_guard.current_version()
382 }
383}
384
385type SenderMap = HashMap<WorkerKey, UnboundedSender<Notification>>;
386
387struct NotificationManagerCore {
388 frontend_senders: SenderMap,
390 hummock_senders: SenderMap,
392 compactor_senders: SenderMap,
394 compute_senders: SenderMap,
396 local_senders: Vec<UnboundedSender<LocalNotification>>,
398 exiting: bool,
399}
400
401impl NotificationManagerCore {
402 fn new() -> Self {
403 Self {
404 frontend_senders: HashMap::new(),
405 hummock_senders: HashMap::new(),
406 compactor_senders: HashMap::new(),
407 compute_senders: HashMap::new(),
408 local_senders: vec![],
409 exiting: false,
410 }
411 }
412
413 fn notify(&mut self, target: Target, response: SubscribeResponse) {
414 macro_rules! warn_send_failure {
415 ($subscribe_type:expr, $worker_key:expr, $err:expr) => {
416 tracing::warn!(
417 "Failed to notify {:?} {:?}: {}",
418 $subscribe_type,
419 $worker_key,
420 $err
421 );
422 };
423 }
424
425 let (subscribe_type, worker_key) = match target {
426 Target::Subscribers {
427 subscribe_type,
428 worker_key,
429 } => (subscribe_type, worker_key),
430 Target::Sender(sender) => {
431 let _ = sender.send(Ok(response)).inspect_err(|err| {
432 tracing::warn!("Failed to send snapshot: {}", err.as_report());
433 });
434 return;
435 }
436 };
437 let senders = self.senders_of(subscribe_type);
438
439 if let Some(worker_key) = worker_key {
440 match senders.entry(worker_key.clone()) {
441 Entry::Occupied(entry) => {
442 let _ = entry.get().send(Ok(response)).inspect_err(|err| {
443 warn_send_failure!(subscribe_type, &worker_key, err.as_report());
444 entry.remove_entry();
445 });
446 }
447 Entry::Vacant(_) => {
448 tracing::warn!("Failed to find notification sender of {:?}", worker_key)
449 }
450 }
451 } else {
452 senders.retain(|worker_key, sender| {
453 sender
454 .send(Ok(response.clone()))
455 .inspect_err(|err| {
456 warn_send_failure!(subscribe_type, &worker_key, err.as_report());
457 })
458 .is_ok()
459 });
460 }
461 }
462
463 fn senders_of(&mut self, subscribe_type: SubscribeType) -> &mut SenderMap {
464 match subscribe_type {
465 SubscribeType::Frontend => &mut self.frontend_senders,
466 SubscribeType::Hummock => &mut self.hummock_senders,
467 SubscribeType::Compactor => &mut self.compactor_senders,
468 SubscribeType::Compute => &mut self.compute_senders,
469 SubscribeType::Unspecified => unreachable!(),
470 }
471 }
472}
473
474#[cfg(test)]
475mod tests {
476 use risingwave_common::id::JobId;
477 use risingwave_pb::common::HostAddress;
478
479 use super::*;
480 use crate::manager::WorkerKey;
481
482 #[tokio::test]
483 async fn test_multiple_subscribers_one_worker() {
484 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
485 let worker_key1 = WorkerKey(HostAddress {
486 host: "a".to_owned(),
487 port: 1,
488 });
489 let worker_key2 = WorkerKey(HostAddress {
490 host: "a".to_owned(),
491 port: 2,
492 });
493 let (tx1, mut rx1) = mpsc::unbounded_channel();
494 let (tx2, mut rx2) = mpsc::unbounded_channel();
495 let (tx3, mut rx3) = mpsc::unbounded_channel();
496 mgr.insert_sender(SubscribeType::Hummock, worker_key1.clone(), tx1.clone());
497 mgr.insert_sender(SubscribeType::Frontend, worker_key1.clone(), tx2);
498 mgr.insert_sender(SubscribeType::Frontend, worker_key2, tx3);
499 mgr.notify_snapshot(tx1, MetaSnapshot::default());
500 assert!(rx1.recv().await.is_some());
501 assert!(rx2.try_recv().is_err());
502 assert!(rx3.try_recv().is_err());
503
504 let version = mgr
505 .notify_frontend(Operation::Add, Info::Database(Default::default()))
506 .await;
507 assert!(rx1.try_recv().is_err());
508 let frontend_notification = rx2.recv().await.unwrap().unwrap();
509 assert_eq!(frontend_notification.version, version);
510 assert_eq!(
511 rx3.recv().await.unwrap().unwrap().version,
512 frontend_notification.version
513 );
514
515 mgr.notify_hummock(Operation::Add, Info::Database(Default::default()));
516 assert_eq!(
517 rx1.recv().await.unwrap().unwrap().version,
518 frontend_notification.version
519 );
520 }
521
522 #[tokio::test]
523 async fn test_frontend_notification_batch_assigns_consecutive_versions() {
524 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
525 let worker_key = WorkerKey(HostAddress {
526 host: "a".to_owned(),
527 port: 1,
528 });
529 let (tx, mut rx) = mpsc::unbounded_channel();
530 mgr.insert_sender(SubscribeType::Frontend, worker_key, tx);
531
532 let initial_version = mgr.current_version().await;
533 let mut batch = FrontendNotificationBatch::new(&mgr);
534 batch.add(Operation::Add, Info::Database(Default::default()));
535 batch.add(Operation::Add, Info::Schema(Default::default()));
536 let published_version = batch
537 .publish()
538 .await
539 .expect("the test notification batch contains two notifications");
540
541 assert_eq!(published_version, initial_version + 2);
542 assert_eq!(mgr.current_version().await, published_version);
543 assert_eq!(
544 rx.recv().await.unwrap().unwrap().version,
545 initial_version + 1
546 );
547 assert_eq!(rx.recv().await.unwrap().unwrap().version, published_version);
548 }
549
550 #[tokio::test]
551 async fn test_notify_hummock_targeted_update() {
552 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
553 let worker_key1 = WorkerKey(HostAddress {
554 host: "a".to_owned(),
555 port: 1,
556 });
557 let worker_key2 = WorkerKey(HostAddress {
558 host: "a".to_owned(),
559 port: 2,
560 });
561 let (hummock_tx1, mut hummock_rx1) = mpsc::unbounded_channel();
562 let (hummock_tx2, mut hummock_rx2) = mpsc::unbounded_channel();
563 let (frontend_tx1, mut frontend_rx1) = mpsc::unbounded_channel();
564 mgr.insert_sender(SubscribeType::Hummock, worker_key1.clone(), hummock_tx1);
565 mgr.insert_sender(SubscribeType::Hummock, worker_key2.clone(), hummock_tx2);
566 mgr.insert_sender(SubscribeType::Frontend, worker_key1.clone(), frontend_tx1);
567
568 let version = mgr.current_version().await;
569 mgr.notify_hummock_targeted_update(worker_key1, Info::Database(Default::default()));
570
571 let notification = hummock_rx1.recv().await.unwrap().unwrap();
572 assert_eq!(notification.version, version);
573 assert_eq!(mgr.current_version().await, version);
574 assert!(hummock_rx2.try_recv().is_err());
575 assert!(frontend_rx1.try_recv().is_err());
576 }
577
578 #[tokio::test]
579 async fn test_snapshot_uses_original_sender_after_resubscribe() {
580 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
581 let worker_key = WorkerKey(HostAddress {
582 host: "a".to_owned(),
583 port: 1,
584 });
585 let (tx1, mut rx1) = mpsc::unbounded_channel();
586 mgr.insert_sender(SubscribeType::Hummock, worker_key.clone(), tx1.clone());
587 mgr.notify_snapshot(tx1, MetaSnapshot::default());
588
589 let (tx2, mut rx2) = mpsc::unbounded_channel();
590 mgr.insert_sender(SubscribeType::Hummock, worker_key, tx2);
591
592 assert!(rx1.recv().await.is_some());
593 assert!(rx2.try_recv().is_err());
594
595 let version = mgr.current_version().await;
596 mgr.notify_hummock(Operation::Add, Info::Database(Default::default()));
597 assert!(rx1.try_recv().is_err());
598 let notification = rx2.recv().await.unwrap().unwrap();
599 assert_eq!(notification.version, version);
600 assert_eq!(mgr.current_version().await, version);
601 }
602
603 #[tokio::test]
604 async fn test_snapshot_arrives_after_notification_it_covers() {
605 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
606 let worker_key = WorkerKey(HostAddress {
607 host: "a".to_owned(),
608 port: 1,
609 });
610 let (tx, mut rx) = mpsc::unbounded_channel();
611 mgr.insert_sender(SubscribeType::Frontend, worker_key, tx.clone());
612
613 let version = mgr
616 .notify_frontend(Operation::Add, Info::Database(Default::default()))
617 .await;
618 mgr.notify_snapshot(tx, MetaSnapshot::default());
619
620 let first = rx.recv().await.unwrap().unwrap();
621 assert!(matches!(first.info, Some(Info::Database(_))));
622 assert_eq!(first.version, version);
623 let second = rx.recv().await.unwrap().unwrap();
624 assert!(matches!(second.info, Some(Info::Snapshot(_))));
625 }
626
627 #[tokio::test]
628 async fn test_local_notification_backfill_finished() {
629 let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
630 let (tx, mut rx) = mpsc::unbounded_channel();
631 mgr.insert_local_sender(tx);
632
633 let job_id = JobId::new(42);
634 mgr.notify_local_subscribers(LocalNotification::StreamingJobBackfillFinished(job_id));
635
636 match rx.recv().await.expect("should receive notification") {
637 LocalNotification::StreamingJobBackfillFinished(received) => {
638 assert_eq!(received, job_id);
639 }
640 other => panic!("unexpected notification: {other:?}"),
641 }
642 }
643}