risingwave_backup/
lib.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(clippy::derive_partial_eq_without_eq)]
16#![feature(trait_alias)]
17#![feature(type_alias_impl_trait)]
18#![feature(custom_test_frameworks)]
19#![feature(map_try_insert)]
20#![feature(error_generic_member_access)]
21#![feature(coverage_attribute)]
22
23pub mod error;
24pub mod meta_snapshot;
25pub mod meta_snapshot_v1;
26pub mod meta_snapshot_v2;
27pub mod storage;
28
29use std::collections::{HashMap, HashSet};
30
31use itertools::Itertools;
32use risingwave_common::RW_VERSION;
33use risingwave_hummock_sdk::state_table_info::StateTableInfo;
34use risingwave_hummock_sdk::version::HummockVersion;
35use risingwave_hummock_sdk::{HummockRawObjectId, HummockVersionId};
36use risingwave_pb::backup_service::{PbMetaSnapshotManifest, PbMetaSnapshotMetadata};
37use risingwave_pb::id::TableId;
38use serde::{Deserialize, Serialize};
39
40use crate::error::{BackupError, BackupResult};
41
42pub type MetaSnapshotId = u64;
43pub type MetaBackupJobId = u64;
44
45/// `MetaSnapshotMetadata` is metadata of `MetaSnapshot`.
46#[derive(Serialize, Deserialize, Clone)]
47pub struct MetaSnapshotMetadata {
48    pub id: MetaSnapshotId,
49    pub hummock_version_id: HummockVersionId,
50    // rename to `ssts` for backward compatibility
51    #[serde(rename = "ssts")]
52    pub objects: HashSet<HummockRawObjectId>,
53    #[serde(default)]
54    pub format_version: u32,
55    pub remarks: Option<String>,
56    #[serde(default)]
57    pub state_table_info: HashMap<TableId, StateTableInfo>,
58    pub rw_version: Option<String>,
59}
60
61impl MetaSnapshotMetadata {
62    pub fn new(
63        id: MetaSnapshotId,
64        v: &HummockVersion,
65        format_version: u32,
66        remarks: Option<String>,
67        table_change_log_object_ids: impl Iterator<Item = HummockRawObjectId>,
68    ) -> Self {
69        Self {
70            id,
71            hummock_version_id: v.id,
72            objects: v
73                .get_object_ids()
74                .map(|object_id| object_id.as_raw())
75                .chain(table_change_log_object_ids)
76                .collect(),
77            format_version,
78            remarks,
79            state_table_info: v
80                .state_table_info
81                .info()
82                .iter()
83                .map(|(id, info)| (*id, info.into()))
84                .collect(),
85            rw_version: Some(RW_VERSION.to_owned()),
86        }
87    }
88}
89
90/// `MetaSnapshotManifest` is the source of truth for valid `MetaSnapshot`.
91#[derive(Serialize, Deserialize, Default, Clone)]
92pub struct MetaSnapshotManifest {
93    pub manifest_id: u64,
94    pub snapshot_metadata: Vec<MetaSnapshotMetadata>,
95}
96
97pub fn xxhash64_checksum(data: &[u8]) -> u64 {
98    twox_hash::XxHash64::oneshot(0, data)
99}
100
101pub fn xxhash64_verify(data: &[u8], checksum: u64) -> BackupResult<()> {
102    let data_checksum = xxhash64_checksum(data);
103    if data_checksum != checksum {
104        return Err(BackupError::ChecksumMismatch {
105            expected: checksum,
106            found: data_checksum,
107        });
108    }
109    Ok(())
110}
111
112impl From<&MetaSnapshotMetadata> for PbMetaSnapshotMetadata {
113    fn from(m: &MetaSnapshotMetadata) -> Self {
114        Self {
115            id: m.id,
116            hummock_version_id: m.hummock_version_id,
117            format_version: Some(m.format_version),
118            remarks: m.remarks.clone(),
119            state_table_info: m
120                .state_table_info
121                .iter()
122                .map(|(t, i)| ((*t), i.into()))
123                .collect(),
124            rw_version: m.rw_version.clone(),
125        }
126    }
127}
128
129impl From<&MetaSnapshotManifest> for PbMetaSnapshotManifest {
130    fn from(m: &MetaSnapshotManifest) -> Self {
131        Self {
132            manifest_id: m.manifest_id,
133            snapshot_metadata: m.snapshot_metadata.iter().map_into().collect_vec(),
134        }
135    }
136}
risingwave_backup/lib.rs

risingwave_backup/
lib.rs
