risingwave_meta/hummock/manager/
context.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, HashMap, HashSet};
16use std::sync::Arc;
17
18use fail::fail_point;
19use futures::{StreamExt, stream};
20use itertools::Itertools;
21use risingwave_common::catalog::TableId;
22use risingwave_hummock_sdk::version::HummockVersion;
23use risingwave_hummock_sdk::{
24    HummockContextId, HummockSstableObjectId, HummockVersionId, INVALID_VERSION_ID,
25    LocalSstableInfo,
26};
27use risingwave_meta_model::hummock_gc_history;
28use risingwave_pb::hummock::{HummockPinnedVersion, ValidationTask};
29use sea_orm::{DatabaseConnection, EntityTrait};
30
31use crate::controller::SqlMetaStore;
32use crate::hummock::HummockManager;
33use crate::hummock::error::{Error, Result};
34use crate::hummock::manager::commit_multi_var;
35use crate::hummock::manager::worker::{HummockManagerEvent, HummockManagerEventSender};
36use crate::hummock::metrics_utils::trigger_pin_unpin_version_state;
37use crate::manager::{META_NODE_ID, MetadataManager};
38use crate::model::BTreeMapTransaction;
39use crate::rpc::metrics::MetaMetrics;
40
41/// `HummockVersionSafePoint` prevents hummock versions GE than it from being GC.
42/// It's used by meta node itself to temporarily pin versions.
43pub struct HummockVersionSafePoint {
44    pub id: HummockVersionId,
45    event_sender: HummockManagerEventSender,
46}
47
48impl Drop for HummockVersionSafePoint {
49    fn drop(&mut self) {
50        if self
51            .event_sender
52            .send(HummockManagerEvent::DropSafePoint(self.id))
53            .is_err()
54        {
55            tracing::debug!("failed to drop hummock version safe point {}", self.id);
56        }
57    }
58}
59
60#[derive(Default)]
61pub(super) struct ContextInfo {
62    pub pinned_versions: BTreeMap<HummockContextId, HummockPinnedVersion>,
63    /// `version_safe_points` is similar to `pinned_versions` expect for being a transient state.
64    pub version_safe_points: Vec<HummockVersionId>,
65}
66
67impl ContextInfo {
68    /// Release resources pinned by these contexts, including:
69    /// - Version
70    /// - Snapshot
71    async fn release_contexts(
72        &mut self,
73        context_ids: impl AsRef<[HummockContextId]>,
74        meta_store_ref: SqlMetaStore,
75    ) -> Result<()> {
76        fail_point!("release_contexts_metastore_err", |_| Err(Error::MetaStore(
77            anyhow::anyhow!("failpoint metastore error")
78        )));
79        fail_point!("release_contexts_internal_err", |_| Err(Error::Internal(
80            anyhow::anyhow!("failpoint internal error")
81        )));
82
83        let mut pinned_versions = BTreeMapTransaction::new(&mut self.pinned_versions);
84        for context_id in context_ids.as_ref() {
85            pinned_versions.remove(*context_id);
86        }
87        commit_multi_var!(meta_store_ref, pinned_versions)?;
88
89        Ok(())
90    }
91}
92
93impl HummockManager {
94    pub async fn release_contexts(
95        &self,
96        context_ids: impl AsRef<[HummockContextId]>,
97    ) -> Result<()> {
98        let mut context_info = self.context_info.write().await;
99        context_info
100            .release_contexts(context_ids, self.env.meta_store())
101            .await?;
102        #[cfg(test)]
103        {
104            drop(context_info);
105            self.check_state_consistency().await;
106        }
107        Ok(())
108    }
109
110    /// Checks whether `context_id` is valid.
111    pub async fn check_context(&self, context_id: HummockContextId) -> Result<bool> {
112        self.context_info
113            .read()
114            .await
115            .check_context(context_id, &self.metadata_manager)
116            .await
117    }
118
119    async fn check_context_with_meta_node(
120        &self,
121        context_id: HummockContextId,
122        context_info: &ContextInfo,
123    ) -> Result<()> {
124        if context_id == META_NODE_ID {
125            // Using the preserved meta id is allowed.
126        } else if !context_info
127            .check_context(context_id, &self.metadata_manager)
128            .await?
129        {
130            // The worker is not found in cluster.
131            return Err(Error::InvalidContext(context_id));
132        }
133        Ok(())
134    }
135
136    #[cfg(any(test, feature = "test"))]
137    pub async fn get_min_pinned_version_id(&self) -> HummockVersionId {
138        self.context_info.read().await.min_pinned_version_id()
139    }
140}
141
142impl ContextInfo {
143    /// Checks whether `context_id` is valid.
144    ///
145    /// Need `&self` to sync with `release_context`
146    pub(super) async fn check_context(
147        &self,
148        context_id: HummockContextId,
149        metadata_manager: &MetadataManager,
150    ) -> Result<bool> {
151        Ok(metadata_manager
152            .get_worker_by_id(context_id)
153            .await
154            .map_err(|err| Error::MetaStore(err.into()))?
155            .is_some())
156    }
157}
158
159impl HummockManager {
160    /// Release invalid contexts, aka worker node ids which are no longer valid in `ClusterManager`.
161    pub(super) async fn release_invalid_contexts(&self) -> Result<Vec<HummockContextId>> {
162        let (active_context_ids, mut context_info) = {
163            let compaction_guard = self.compaction.read().await;
164            let context_info = self.context_info.write().await;
165            let mut active_context_ids = HashSet::new();
166            active_context_ids.extend(
167                compaction_guard
168                    .compact_task_assignment
169                    .values()
170                    .map(|c| c.context_id),
171            );
172            active_context_ids.extend(context_info.pinned_versions.keys());
173            (active_context_ids, context_info)
174        };
175
176        let mut invalid_context_ids = vec![];
177        for active_context_id in &active_context_ids {
178            if !context_info
179                .check_context(*active_context_id, &self.metadata_manager)
180                .await?
181            {
182                invalid_context_ids.push(*active_context_id);
183            }
184        }
185
186        context_info
187            .release_contexts(&invalid_context_ids, self.env.meta_store())
188            .await?;
189
190        Ok(invalid_context_ids)
191    }
192
193    pub async fn commit_epoch_sanity_check(
194        &self,
195        tables_to_commit: &HashMap<TableId, u64>,
196        sstables: &[LocalSstableInfo],
197        sst_to_context: &HashMap<HummockSstableObjectId, HummockContextId>,
198        current_version: &HummockVersion,
199    ) -> Result<()> {
200        use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
201
202        for (sst_id, context_id) in sst_to_context {
203            #[cfg(test)]
204            {
205                if *context_id == crate::manager::META_NODE_ID {
206                    continue;
207                }
208            }
209            if !self
210                .context_info
211                .read()
212                .await
213                .check_context(*context_id, &self.metadata_manager)
214                .await?
215            {
216                return Err(Error::InvalidSst(*sst_id));
217            }
218        }
219
220        // sanity check on monotonically increasing table committed epoch
221        for (table_id, committed_epoch) in tables_to_commit {
222            if let Some(info) = current_version.state_table_info.info().get(table_id)
223                && *committed_epoch <= info.committed_epoch
224            {
225                return Err(anyhow::anyhow!(
226                    "table {} Epoch {} <= committed_epoch {}",
227                    table_id,
228                    committed_epoch,
229                    info.committed_epoch,
230                )
231                .into());
232            }
233        }
234
235        // HummockManager::now requires a write to the meta store. Thus, it should be avoided whenever feasible.
236        if !sstables.is_empty() {
237            // Sanity check to ensure SSTs to commit have not been full GCed yet.
238            let now = self.now().await?;
239            check_sst_retention(
240                now,
241                self.env.opts.min_sst_retention_time_sec,
242                sstables
243                    .iter()
244                    .map(|s| (s.sst_info.object_id, s.created_at)),
245            )?;
246            if self.env.opts.gc_history_retention_time_sec != 0 {
247                let ids = sstables.iter().map(|s| s.sst_info.object_id).collect_vec();
248                check_gc_history(&self.meta_store_ref().conn, ids).await?;
249            }
250        }
251
252        async {
253            if !self.env.opts.enable_committed_sst_sanity_check {
254                return;
255            }
256            if sstables.is_empty() {
257                return;
258            }
259            let compactor = match self.compactor_manager.next_compactor() {
260                None => {
261                    tracing::warn!("Skip committed SST sanity check due to no available worker");
262                    return;
263                }
264                Some(compactor) => compactor,
265            };
266            let sst_infos = sstables
267                .iter()
268                .map(|LocalSstableInfo { sst_info, .. }| sst_info.clone())
269                .collect_vec();
270            if compactor
271                .send_event(
272                    #[expect(deprecated)]
273                    ResponseEvent::ValidationTask(ValidationTask {
274                        sst_infos: sst_infos.into_iter().map(|sst| sst.into()).collect_vec(),
275                        sst_id_to_worker_id: sst_to_context
276                            .iter()
277                            .map(|(object_id, worker_id)| (*object_id, *worker_id))
278                            .collect(),
279                    }),
280                )
281                .is_err()
282            {
283                tracing::warn!("Skip committed SST sanity check due to send failure");
284            }
285        }
286        .await;
287        Ok(())
288    }
289
290    pub async fn release_meta_context(&self) -> Result<()> {
291        self.release_contexts([META_NODE_ID]).await
292    }
293
294    pub(crate) async fn report_compaction_sanity_check(
295        &self,
296        object_timestamps: &HashMap<HummockSstableObjectId, u64>,
297    ) -> Result<()> {
298        // HummockManager::now requires a write to the meta store. Thus, it should be avoided whenever feasible.
299        if object_timestamps.is_empty() {
300            return Ok(());
301        }
302        let now = self.now().await?;
303        check_sst_retention(
304            now,
305            self.env.opts.min_sst_retention_time_sec,
306            object_timestamps.iter().map(|(k, v)| (*k, *v)),
307        )?;
308        if self.env.opts.gc_history_retention_time_sec != 0 {
309            let ids = object_timestamps.keys().copied().collect_vec();
310            check_gc_history(&self.meta_store_ref().conn, ids).await?;
311        }
312        Ok(())
313    }
314}
315
316fn check_sst_retention(
317    now: u64,
318    retention_sec: u64,
319    sst_infos: impl Iterator<Item = (HummockSstableObjectId, u64)>,
320) -> Result<()> {
321    let sst_retention_watermark = now.saturating_sub(retention_sec);
322    for (object_id, created_at) in sst_infos {
323        if created_at < sst_retention_watermark {
324            return Err(anyhow::anyhow!("object {object_id} is rejected from being committed since it's below watermark: object timestamp {created_at}, meta node timestamp {now}, retention_sec {retention_sec}, watermark {sst_retention_watermark}").into());
325        }
326    }
327    Ok(())
328}
329
330async fn check_gc_history(
331    db: &DatabaseConnection,
332    // need IntoIterator to work around stream's "implementation of `std::iter::Iterator` is not general enough" error.
333    object_ids: impl IntoIterator<Item = HummockSstableObjectId>,
334) -> Result<()> {
335    let futures = object_ids.into_iter().map(|id| async move {
336        hummock_gc_history::Entity::find_by_id(id)
337            .one(db)
338            .await
339            .map_err(Error::from)
340    });
341    let res: Vec<_> = stream::iter(futures).buffer_unordered(10).collect().await;
342    let res: Result<Vec<_>> = res.into_iter().collect();
343    let mut expired_object_ids = res?.into_iter().flatten().peekable();
344    if expired_object_ids.peek().is_none() {
345        return Ok(());
346    }
347    let expired_object_ids: Vec<_> = expired_object_ids.collect();
348    tracing::error!(
349        ?expired_object_ids,
350        "new SSTs are rejected because they have already been GCed"
351    );
352    Err(Error::InvalidSst(expired_object_ids[0].object_id))
353}
354
355// pin and unpin method
356impl HummockManager {
357    /// Pin the current greatest hummock version. The pin belongs to `context_id`
358    /// and will be unpinned when `context_id` is invalidated.
359    pub async fn pin_version(&self, context_id: HummockContextId) -> Result<Arc<HummockVersion>> {
360        let versioning = self.versioning.read().await;
361        let mut context_info = self.context_info.write().await;
362        self.check_context_with_meta_node(context_id, &context_info)
363            .await?;
364        let mut pinned_versions = BTreeMapTransaction::new(&mut context_info.pinned_versions);
365        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
366            context_id,
367            HummockPinnedVersion {
368                context_id,
369                min_pinned_id: INVALID_VERSION_ID,
370            },
371        );
372        let version_id = versioning.current_version.id;
373        let ret = versioning.current_version.clone();
374        if context_pinned_version.min_pinned_id == INVALID_VERSION_ID
375            || context_pinned_version.min_pinned_id > version_id
376        {
377            context_pinned_version.min_pinned_id = version_id;
378            commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
379            trigger_pin_unpin_version_state(&self.metrics, &context_info.pinned_versions);
380        }
381
382        #[cfg(test)]
383        {
384            drop(context_info);
385            drop(versioning);
386            self.check_state_consistency().await;
387        }
388
389        Ok(ret)
390    }
391
392    /// Unpin all pins which belongs to `context_id` and has an id which is older than
393    /// `unpin_before`. All versions >= `unpin_before` will be treated as if they are all pinned by
394    /// this `context_id` so they will not be vacuumed.
395    pub async fn unpin_version_before(
396        &self,
397        context_id: HummockContextId,
398        unpin_before: HummockVersionId,
399    ) -> Result<()> {
400        let mut context_info = self.context_info.write().await;
401        self.check_context_with_meta_node(context_id, &context_info)
402            .await?;
403        let mut pinned_versions = BTreeMapTransaction::new(&mut context_info.pinned_versions);
404        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
405            context_id,
406            HummockPinnedVersion {
407                context_id,
408                min_pinned_id: HummockVersionId::default(),
409            },
410        );
411        assert!(
412            context_pinned_version.min_pinned_id <= unpin_before,
413            "val must be monotonically non-decreasing. old = {}, new = {}.",
414            context_pinned_version.min_pinned_id,
415            unpin_before
416        );
417        context_pinned_version.min_pinned_id = unpin_before;
418        commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
419        trigger_pin_unpin_version_state(&self.metrics, &context_info.pinned_versions);
420
421        #[cfg(test)]
422        {
423            drop(context_info);
424            self.check_state_consistency().await;
425        }
426
427        Ok(())
428    }
429}
430
431// safe point
432impl HummockManager {
433    pub async fn register_safe_point(&self) -> HummockVersionSafePoint {
434        let versioning = self.versioning.read().await;
435        let mut wl = self.context_info.write().await;
436        let safe_point = HummockVersionSafePoint {
437            id: versioning.current_version.id,
438            event_sender: self.event_sender.clone(),
439        };
440        wl.version_safe_points.push(safe_point.id);
441        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
442        safe_point
443    }
444
445    pub async fn unregister_safe_point(&self, safe_point: HummockVersionId) {
446        let mut wl = self.context_info.write().await;
447        let version_safe_points = &mut wl.version_safe_points;
448        if let Some(pos) = version_safe_points.iter().position(|sp| *sp == safe_point) {
449            version_safe_points.remove(pos);
450        }
451        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
452    }
453}
454
455fn trigger_safepoint_stat(metrics: &MetaMetrics, safepoints: &[HummockVersionId]) {
456    if let Some(sp) = safepoints.iter().min() {
457        metrics.min_safepoint_version_id.set(sp.as_i64_id());
458    } else {
459        metrics.min_safepoint_version_id.set(u64::MAX as _);
460    }
461}
risingwave_meta/hummock/manager/context.rs

risingwave_meta/hummock/manager/
context.rs
