Skip to main content

risingwave_meta/manager/
notification.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::collections::hash_map::Entry;
17use std::sync::Arc;
18
19use risingwave_common::id::JobId;
20use risingwave_common::system_param::reader::SystemParamsReader;
21use risingwave_common::util::iter_util::ZipEqDebug;
22use risingwave_meta_model::ObjectId;
23use risingwave_pb::common::{WorkerNode, WorkerType};
24use risingwave_pb::meta::object::PbObjectInfo;
25use risingwave_pb::meta::subscribe_response::{Info, Operation};
26use risingwave_pb::meta::{
27    MetaSnapshot, PbObject, PbObjectGroup, SubscribeResponse, SubscribeType,
28};
29use thiserror_ext::AsReport;
30use tokio::sync::Mutex;
31use tokio::sync::mpsc::{self, UnboundedSender};
32use tonic::Status;
33
34use crate::controller::SqlMetaStore;
35use crate::manager::WorkerKey;
36use crate::manager::notification_version::NotificationVersionGenerator;
37use crate::model::FragmentId;
38
39pub type MessageStatus = Status;
40pub type Notification = Result<SubscribeResponse, Status>;
41pub type NotificationManagerRef = Arc<NotificationManager>;
42pub type NotificationVersion = u64;
43/// NOTE(kwannoel): This is just ignored, used in background DDL
44pub const IGNORED_NOTIFICATION_VERSION: u64 = 0;
45
46/// A batch of frontend notifications that belong to the same catalog update.
47///
48/// Publishing the batch assigns consecutive catalog notification versions to its notifications.
49/// Therefore, waiting for the returned final version also waits for every earlier notification in
50/// the batch to be processed.
51pub struct FrontendNotificationBatch<'a> {
52    notification_manager: &'a NotificationManager,
53    pending: Vec<(Operation, Info)>,
54}
55
56impl<'a> FrontendNotificationBatch<'a> {
57    pub fn new(notification_manager: &'a NotificationManager) -> Self {
58        Self {
59            notification_manager,
60            pending: Vec::new(),
61        }
62    }
63
64    pub fn add(&mut self, operation: Operation, info: Info) {
65        self.pending.push((operation, info));
66    }
67
68    #[must_use]
69    pub async fn publish(self) -> Option<NotificationVersion> {
70        self.notification_manager
71            .notify_frontend_batch(self.pending)
72            .await
73    }
74}
75
76#[derive(Clone, Debug)]
77pub enum LocalNotification {
78    WorkerNodeDeleted(WorkerNode),
79    WorkerNodeActivated(WorkerNode),
80    SystemParamsChange(SystemParamsReader),
81    BatchParallelismChange,
82    ServingFragmentMappingsUpsert(Vec<FragmentId>),
83    ServingFragmentMappingsDelete(Vec<FragmentId>),
84    SourceDropped(ObjectId),
85    StreamingJobBackfillFinished(JobId),
86}
87
88#[derive(Debug)]
89enum Target {
90    Subscribers {
91        subscribe_type: SubscribeType,
92        // `None` indicates sending to all subscribers of `subscribe_type`.
93        worker_key: Option<WorkerKey>,
94    },
95    /// The sender of one subscribe call. By the time the task is delivered, the worker may have
96    /// subscribed again and replaced its entry in the sender map.
97    Sender(UnboundedSender<Notification>),
98}
99
100impl From<SubscribeType> for Target {
101    fn from(value: SubscribeType) -> Self {
102        Self::Subscribers {
103            subscribe_type: value,
104            worker_key: None,
105        }
106    }
107}
108
109#[derive(Debug)]
110struct Task {
111    target: Target,
112    operation: Operation,
113    info: Info,
114    /// `None` means using the latest version published to the notification queue without
115    /// incrementing it.
116    version: Option<NotificationVersion>,
117}
118
119/// [`NotificationManager`] is used to send notification to frontends and compute nodes.
120pub struct NotificationManager {
121    core: Arc<parking_lot::Mutex<NotificationManagerCore>>,
122    /// Sender used to add a notification into the waiting queue.
123    task_tx: UnboundedSender<Task>,
124    /// The current notification version generator.
125    version_generator: Mutex<NotificationVersionGenerator>,
126}
127
128impl NotificationManager {
129    pub async fn new(meta_store_impl: SqlMetaStore) -> Self {
130        // notification waiting queue.
131        let (task_tx, mut task_rx) = mpsc::unbounded_channel::<Task>();
132        let core = Arc::new(parking_lot::Mutex::new(NotificationManagerCore::new()));
133        let core_clone = core.clone();
134        let version_generator = NotificationVersionGenerator::new(meta_store_impl)
135            .await
136            .unwrap();
137        let mut latest_version = version_generator.current_version();
138
139        tokio::spawn(async move {
140            while let Some(task) = task_rx.recv().await {
141                if let Some(version) = task.version {
142                    latest_version = version;
143                }
144                let response = SubscribeResponse {
145                    status: None,
146                    operation: task.operation as i32,
147                    info: Some(task.info),
148                    version: latest_version,
149                };
150                core.lock().notify(task.target, response);
151            }
152        });
153
154        Self {
155            core: core_clone,
156            task_tx,
157            version_generator: Mutex::new(version_generator),
158        }
159    }
160
161    pub fn abort_all(&self) {
162        let mut guard = self.core.lock();
163        *guard = NotificationManagerCore::new();
164        guard.exiting = true;
165    }
166
167    #[inline(always)]
168    fn notify(
169        &self,
170        target: Target,
171        operation: Operation,
172        info: Info,
173        version: Option<NotificationVersion>,
174    ) {
175        let task = Task {
176            target,
177            operation,
178            info,
179            version,
180        };
181        self.task_tx.send(task).unwrap();
182    }
183
184    /// Add a notification to the waiting queue and increase notification version.
185    async fn notify_with_version(
186        &self,
187        target: Target,
188        operation: Operation,
189        info: Info,
190    ) -> NotificationVersion {
191        let mut version_guard = self.version_generator.lock().await;
192        version_guard.increase_version().await;
193        let version = version_guard.current_version();
194        self.notify(target, operation, info, Some(version));
195        version
196    }
197
198    /// Assign consecutive versions to a frontend notification batch and enqueue the complete batch
199    /// while holding the version generator lock.
200    async fn notify_frontend_batch(
201        &self,
202        notifications: Vec<(Operation, Info)>,
203    ) -> Option<NotificationVersion> {
204        let batch_len: NotificationVersion = notifications.len().try_into().unwrap();
205        if batch_len == 0 {
206            return None;
207        }
208
209        let mut version_guard = self.version_generator.lock().await;
210        let first_version = version_guard.current_version() + 1;
211        version_guard.increase_version_by(batch_len).await;
212        let final_version = version_guard.current_version();
213
214        for ((operation, info), version) in notifications
215            .into_iter()
216            .zip_eq_debug(first_version..=final_version)
217        {
218            self.notify(
219                SubscribeType::Frontend.into(),
220                operation,
221                info,
222                Some(version),
223            );
224        }
225
226        Some(final_version)
227    }
228
229    /// Add a notification using the latest version published to the queue without incrementing it.
230    #[inline(always)]
231    fn notify_without_version(&self, target: Target, operation: Operation, info: Info) {
232        self.notify(target, operation, info, None);
233    }
234
235    /// Queue the snapshot behind every notification published before it was read. The observer
236    /// manager filters out such notifications only if they arrive before the snapshot.
237    pub fn notify_snapshot(
238        &self,
239        sender: UnboundedSender<Notification>,
240        meta_snapshot: MetaSnapshot,
241    ) {
242        self.notify_without_version(
243            Target::Sender(sender),
244            Operation::Snapshot,
245            Info::Snapshot(meta_snapshot),
246        );
247    }
248
249    pub fn notify_all_without_version(&self, operation: Operation, info: Info) {
250        for subscribe_type in [
251            SubscribeType::Frontend,
252            SubscribeType::Hummock,
253            SubscribeType::Compactor,
254            SubscribeType::Compute,
255        ] {
256            self.notify_without_version(subscribe_type.into(), operation, info.clone());
257        }
258    }
259
260    #[must_use]
261    pub async fn notify_frontend(&self, operation: Operation, info: Info) -> NotificationVersion {
262        self.notify_with_version(SubscribeType::Frontend.into(), operation, info)
263            .await
264    }
265
266    #[must_use]
267    pub async fn notify_frontend_object_info(
268        &self,
269        operation: Operation,
270        object_info: PbObjectInfo,
271    ) -> NotificationVersion {
272        self.notify_with_version(
273            SubscribeType::Frontend.into(),
274            operation,
275            Info::ObjectGroup(PbObjectGroup {
276                objects: vec![PbObject {
277                    object_info: object_info.into(),
278                }],
279                dependencies: vec![],
280            }),
281        )
282        .await
283    }
284
285    pub(crate) fn notify_hummock_targeted_update(&self, worker_key: WorkerKey, info: Info) {
286        self.notify_without_version(
287            Target::Subscribers {
288                subscribe_type: SubscribeType::Hummock,
289                worker_key: Some(worker_key),
290            },
291            Operation::Update,
292            info,
293        );
294    }
295
296    pub fn notify_compute_without_version(&self, operation: Operation, info: Info) {
297        self.notify_without_version(SubscribeType::Compute.into(), operation, info)
298    }
299
300    pub fn notify_frontend_without_version(&self, operation: Operation, info: Info) {
301        self.notify_without_version(SubscribeType::Frontend.into(), operation, info)
302    }
303
304    pub fn notify_hummock(&self, operation: Operation, info: Info) {
305        self.notify_without_version(SubscribeType::Hummock.into(), operation, info)
306    }
307
308    pub fn notify_compactor(&self, operation: Operation, info: Info) {
309        self.notify_without_version(SubscribeType::Compactor.into(), operation, info)
310    }
311
312    #[cfg(any(test, feature = "test"))]
313    pub fn notify_hummock_with_version(
314        &self,
315        operation: Operation,
316        info: Info,
317        version: Option<NotificationVersion>,
318    ) {
319        self.notify(SubscribeType::Hummock.into(), operation, info, version)
320    }
321
322    pub fn notify_local_subscribers(&self, notification: LocalNotification) {
323        let mut core_guard = self.core.lock();
324        core_guard.local_senders.retain(|sender| {
325            if let Err(err) = sender.send(notification.clone()) {
326                tracing::warn!(error = %err.as_report(), "Failed to notify local subscriber");
327                return false;
328            }
329            true
330        });
331    }
332
333    /// Tell `NotificationManagerCore` to delete sender.
334    pub fn delete_sender(&self, worker_type: WorkerType, worker_key: WorkerKey) {
335        let mut core_guard = self.core.lock();
336        // TODO: we may avoid passing the worker_type and remove the `worker_key` in all sender
337        // holders anyway
338        match worker_type {
339            WorkerType::Frontend => core_guard.frontend_senders.remove(&worker_key),
340            WorkerType::ComputeNode | WorkerType::RiseCtl => {
341                core_guard.hummock_senders.remove(&worker_key)
342            }
343            WorkerType::Compactor => core_guard.compactor_senders.remove(&worker_key),
344            _ => unreachable!(),
345        };
346    }
347
348    /// Tell `NotificationManagerCore` to insert sender by `worker_type`.
349    pub fn insert_sender(
350        &self,
351        subscribe_type: SubscribeType,
352        worker_key: WorkerKey,
353        sender: UnboundedSender<Notification>,
354    ) {
355        let mut core_guard = self.core.lock();
356        if core_guard.exiting {
357            tracing::warn!("notification manager exiting.");
358            return;
359        }
360        let senders = core_guard.senders_of(subscribe_type);
361
362        senders.insert(worker_key, sender);
363    }
364
365    pub fn insert_local_sender(&self, sender: UnboundedSender<LocalNotification>) {
366        let mut core_guard = self.core.lock();
367        if core_guard.exiting {
368            tracing::warn!("notification manager exiting.");
369            return;
370        }
371        core_guard.local_senders.push(sender);
372    }
373
374    #[cfg(test)]
375    pub fn clear_local_sender(&self) {
376        self.core.lock().local_senders.clear();
377    }
378
379    pub async fn current_version(&self) -> NotificationVersion {
380        let version_guard = self.version_generator.lock().await;
381        version_guard.current_version()
382    }
383}
384
385type SenderMap = HashMap<WorkerKey, UnboundedSender<Notification>>;
386
387struct NotificationManagerCore {
388    /// The notification sender to frontends.
389    frontend_senders: SenderMap,
390    /// The notification sender to nodes that subscribes the hummock.
391    hummock_senders: SenderMap,
392    /// The notification sender to compactor nodes.
393    compactor_senders: SenderMap,
394    /// The notification sender to compute nodes.
395    compute_senders: SenderMap,
396    /// The notification sender to local subscribers.
397    local_senders: Vec<UnboundedSender<LocalNotification>>,
398    exiting: bool,
399}
400
401impl NotificationManagerCore {
402    fn new() -> Self {
403        Self {
404            frontend_senders: HashMap::new(),
405            hummock_senders: HashMap::new(),
406            compactor_senders: HashMap::new(),
407            compute_senders: HashMap::new(),
408            local_senders: vec![],
409            exiting: false,
410        }
411    }
412
413    fn notify(&mut self, target: Target, response: SubscribeResponse) {
414        macro_rules! warn_send_failure {
415            ($subscribe_type:expr, $worker_key:expr, $err:expr) => {
416                tracing::warn!(
417                    "Failed to notify {:?} {:?}: {}",
418                    $subscribe_type,
419                    $worker_key,
420                    $err
421                );
422            };
423        }
424
425        let (subscribe_type, worker_key) = match target {
426            Target::Subscribers {
427                subscribe_type,
428                worker_key,
429            } => (subscribe_type, worker_key),
430            Target::Sender(sender) => {
431                let _ = sender.send(Ok(response)).inspect_err(|err| {
432                    tracing::warn!("Failed to send snapshot: {}", err.as_report());
433                });
434                return;
435            }
436        };
437        let senders = self.senders_of(subscribe_type);
438
439        if let Some(worker_key) = worker_key {
440            match senders.entry(worker_key.clone()) {
441                Entry::Occupied(entry) => {
442                    let _ = entry.get().send(Ok(response)).inspect_err(|err| {
443                        warn_send_failure!(subscribe_type, &worker_key, err.as_report());
444                        entry.remove_entry();
445                    });
446                }
447                Entry::Vacant(_) => {
448                    tracing::warn!("Failed to find notification sender of {:?}", worker_key)
449                }
450            }
451        } else {
452            senders.retain(|worker_key, sender| {
453                sender
454                    .send(Ok(response.clone()))
455                    .inspect_err(|err| {
456                        warn_send_failure!(subscribe_type, &worker_key, err.as_report());
457                    })
458                    .is_ok()
459            });
460        }
461    }
462
463    fn senders_of(&mut self, subscribe_type: SubscribeType) -> &mut SenderMap {
464        match subscribe_type {
465            SubscribeType::Frontend => &mut self.frontend_senders,
466            SubscribeType::Hummock => &mut self.hummock_senders,
467            SubscribeType::Compactor => &mut self.compactor_senders,
468            SubscribeType::Compute => &mut self.compute_senders,
469            SubscribeType::Unspecified => unreachable!(),
470        }
471    }
472}
473
474#[cfg(test)]
475mod tests {
476    use risingwave_common::id::JobId;
477    use risingwave_pb::common::HostAddress;
478
479    use super::*;
480    use crate::manager::WorkerKey;
481
482    #[tokio::test]
483    async fn test_multiple_subscribers_one_worker() {
484        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
485        let worker_key1 = WorkerKey(HostAddress {
486            host: "a".to_owned(),
487            port: 1,
488        });
489        let worker_key2 = WorkerKey(HostAddress {
490            host: "a".to_owned(),
491            port: 2,
492        });
493        let (tx1, mut rx1) = mpsc::unbounded_channel();
494        let (tx2, mut rx2) = mpsc::unbounded_channel();
495        let (tx3, mut rx3) = mpsc::unbounded_channel();
496        mgr.insert_sender(SubscribeType::Hummock, worker_key1.clone(), tx1.clone());
497        mgr.insert_sender(SubscribeType::Frontend, worker_key1.clone(), tx2);
498        mgr.insert_sender(SubscribeType::Frontend, worker_key2, tx3);
499        mgr.notify_snapshot(tx1, MetaSnapshot::default());
500        assert!(rx1.recv().await.is_some());
501        assert!(rx2.try_recv().is_err());
502        assert!(rx3.try_recv().is_err());
503
504        let version = mgr
505            .notify_frontend(Operation::Add, Info::Database(Default::default()))
506            .await;
507        assert!(rx1.try_recv().is_err());
508        let frontend_notification = rx2.recv().await.unwrap().unwrap();
509        assert_eq!(frontend_notification.version, version);
510        assert_eq!(
511            rx3.recv().await.unwrap().unwrap().version,
512            frontend_notification.version
513        );
514
515        mgr.notify_hummock(Operation::Add, Info::Database(Default::default()));
516        assert_eq!(
517            rx1.recv().await.unwrap().unwrap().version,
518            frontend_notification.version
519        );
520    }
521
522    #[tokio::test]
523    async fn test_frontend_notification_batch_assigns_consecutive_versions() {
524        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
525        let worker_key = WorkerKey(HostAddress {
526            host: "a".to_owned(),
527            port: 1,
528        });
529        let (tx, mut rx) = mpsc::unbounded_channel();
530        mgr.insert_sender(SubscribeType::Frontend, worker_key, tx);
531
532        let initial_version = mgr.current_version().await;
533        let mut batch = FrontendNotificationBatch::new(&mgr);
534        batch.add(Operation::Add, Info::Database(Default::default()));
535        batch.add(Operation::Add, Info::Schema(Default::default()));
536        let published_version = batch
537            .publish()
538            .await
539            .expect("the test notification batch contains two notifications");
540
541        assert_eq!(published_version, initial_version + 2);
542        assert_eq!(mgr.current_version().await, published_version);
543        assert_eq!(
544            rx.recv().await.unwrap().unwrap().version,
545            initial_version + 1
546        );
547        assert_eq!(rx.recv().await.unwrap().unwrap().version, published_version);
548    }
549
550    #[tokio::test]
551    async fn test_notify_hummock_targeted_update() {
552        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
553        let worker_key1 = WorkerKey(HostAddress {
554            host: "a".to_owned(),
555            port: 1,
556        });
557        let worker_key2 = WorkerKey(HostAddress {
558            host: "a".to_owned(),
559            port: 2,
560        });
561        let (hummock_tx1, mut hummock_rx1) = mpsc::unbounded_channel();
562        let (hummock_tx2, mut hummock_rx2) = mpsc::unbounded_channel();
563        let (frontend_tx1, mut frontend_rx1) = mpsc::unbounded_channel();
564        mgr.insert_sender(SubscribeType::Hummock, worker_key1.clone(), hummock_tx1);
565        mgr.insert_sender(SubscribeType::Hummock, worker_key2.clone(), hummock_tx2);
566        mgr.insert_sender(SubscribeType::Frontend, worker_key1.clone(), frontend_tx1);
567
568        let version = mgr.current_version().await;
569        mgr.notify_hummock_targeted_update(worker_key1, Info::Database(Default::default()));
570
571        let notification = hummock_rx1.recv().await.unwrap().unwrap();
572        assert_eq!(notification.version, version);
573        assert_eq!(mgr.current_version().await, version);
574        assert!(hummock_rx2.try_recv().is_err());
575        assert!(frontend_rx1.try_recv().is_err());
576    }
577
578    #[tokio::test]
579    async fn test_snapshot_uses_original_sender_after_resubscribe() {
580        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
581        let worker_key = WorkerKey(HostAddress {
582            host: "a".to_owned(),
583            port: 1,
584        });
585        let (tx1, mut rx1) = mpsc::unbounded_channel();
586        mgr.insert_sender(SubscribeType::Hummock, worker_key.clone(), tx1.clone());
587        mgr.notify_snapshot(tx1, MetaSnapshot::default());
588
589        let (tx2, mut rx2) = mpsc::unbounded_channel();
590        mgr.insert_sender(SubscribeType::Hummock, worker_key, tx2);
591
592        assert!(rx1.recv().await.is_some());
593        assert!(rx2.try_recv().is_err());
594
595        let version = mgr.current_version().await;
596        mgr.notify_hummock(Operation::Add, Info::Database(Default::default()));
597        assert!(rx1.try_recv().is_err());
598        let notification = rx2.recv().await.unwrap().unwrap();
599        assert_eq!(notification.version, version);
600        assert_eq!(mgr.current_version().await, version);
601    }
602
603    #[tokio::test]
604    async fn test_snapshot_arrives_after_notification_it_covers() {
605        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
606        let worker_key = WorkerKey(HostAddress {
607            host: "a".to_owned(),
608            port: 1,
609        });
610        let (tx, mut rx) = mpsc::unbounded_channel();
611        mgr.insert_sender(SubscribeType::Frontend, worker_key, tx.clone());
612
613        // The test runtime has one thread and `notify_frontend` does not yield after queueing, so
614        // the notification is still in the queue when the snapshot is sent.
615        let version = mgr
616            .notify_frontend(Operation::Add, Info::Database(Default::default()))
617            .await;
618        mgr.notify_snapshot(tx, MetaSnapshot::default());
619
620        let first = rx.recv().await.unwrap().unwrap();
621        assert!(matches!(first.info, Some(Info::Database(_))));
622        assert_eq!(first.version, version);
623        let second = rx.recv().await.unwrap().unwrap();
624        assert!(matches!(second.info, Some(Info::Snapshot(_))));
625    }
626
627    #[tokio::test]
628    async fn test_local_notification_backfill_finished() {
629        let mgr = NotificationManager::new(SqlMetaStore::for_test().await).await;
630        let (tx, mut rx) = mpsc::unbounded_channel();
631        mgr.insert_local_sender(tx);
632
633        let job_id = JobId::new(42);
634        mgr.notify_local_subscribers(LocalNotification::StreamingJobBackfillFinished(job_id));
635
636        match rx.recv().await.expect("should receive notification") {
637            LocalNotification::StreamingJobBackfillFinished(received) => {
638                assert_eq!(received, job_id);
639            }
640            other => panic!("unexpected notification: {other:?}"),
641        }
642    }
643}