1#![allow(clippy::derive_partial_eq_without_eq)]
16#![feature(trait_alias)]
17#![feature(type_alias_impl_trait)]
18#![feature(custom_test_frameworks)]
19#![feature(map_try_insert)]
20#![feature(btree_extract_if)]
21#![feature(let_chains)]
22#![feature(error_generic_member_access)]
23#![feature(coverage_attribute)]
24
25pub mod error;
26pub mod meta_snapshot;
27pub mod meta_snapshot_v1;
28pub mod meta_snapshot_v2;
29pub mod storage;
30
31use std::collections::{HashMap, HashSet};
32
33use itertools::Itertools;
34use risingwave_common::RW_VERSION;
35use risingwave_hummock_sdk::state_table_info::StateTableInfo;
36use risingwave_hummock_sdk::version::HummockVersion;
37use risingwave_hummock_sdk::{HummockRawObjectId, HummockVersionId};
38use risingwave_pb::backup_service::{PbMetaSnapshotManifest, PbMetaSnapshotMetadata};
39use serde::{Deserialize, Serialize};
40
41use crate::error::{BackupError, BackupResult};
42
43pub type MetaSnapshotId = u64;
44pub type MetaBackupJobId = u64;
45
46#[derive(Serialize, Deserialize, Clone)]
48pub struct MetaSnapshotMetadata {
49 pub id: MetaSnapshotId,
50 pub hummock_version_id: HummockVersionId,
51 #[serde(rename = "ssts")]
53 pub objects: HashSet<HummockRawObjectId>,
54 #[serde(default)]
55 pub format_version: u32,
56 pub remarks: Option<String>,
57 #[serde(default)]
58 pub state_table_info: HashMap<u32, StateTableInfo>,
59 pub rw_version: Option<String>,
60}
61
62impl MetaSnapshotMetadata {
63 pub fn new(
64 id: MetaSnapshotId,
65 v: &HummockVersion,
66 format_version: u32,
67 remarks: Option<String>,
68 ) -> Self {
69 Self {
70 id,
71 hummock_version_id: v.id,
72 objects: v
73 .get_object_ids(false)
74 .map(|object_id| object_id.as_raw())
75 .collect(),
76 format_version,
77 remarks,
78 state_table_info: v
79 .state_table_info
80 .info()
81 .iter()
82 .map(|(id, info)| (id.table_id, info.into()))
83 .collect(),
84 rw_version: Some(RW_VERSION.to_owned()),
85 }
86 }
87}
88
89#[derive(Serialize, Deserialize, Default, Clone)]
91pub struct MetaSnapshotManifest {
92 pub manifest_id: u64,
93 pub snapshot_metadata: Vec<MetaSnapshotMetadata>,
94}
95
96pub fn xxhash64_checksum(data: &[u8]) -> u64 {
97 twox_hash::XxHash64::oneshot(0, data)
98}
99
100pub fn xxhash64_verify(data: &[u8], checksum: u64) -> BackupResult<()> {
101 let data_checksum = xxhash64_checksum(data);
102 if data_checksum != checksum {
103 return Err(BackupError::ChecksumMismatch {
104 expected: checksum,
105 found: data_checksum,
106 });
107 }
108 Ok(())
109}
110
111impl From<&MetaSnapshotMetadata> for PbMetaSnapshotMetadata {
112 fn from(m: &MetaSnapshotMetadata) -> Self {
113 Self {
114 id: m.id,
115 hummock_version_id: m.hummock_version_id.to_u64(),
116 format_version: Some(m.format_version),
117 remarks: m.remarks.clone(),
118 state_table_info: m
119 .state_table_info
120 .iter()
121 .map(|(t, i)| (*t, i.into()))
122 .collect(),
123 rw_version: m.rw_version.clone(),
124 }
125 }
126}
127
128impl From<&MetaSnapshotManifest> for PbMetaSnapshotManifest {
129 fn from(m: &MetaSnapshotManifest) -> Self {
130 Self {
131 manifest_id: m.manifest_id,
132 snapshot_metadata: m.snapshot_metadata.iter().map_into().collect_vec(),
133 }
134 }
135}