risingwave_meta/backup_restore/
backup_manager.rs1use std::collections::HashSet;
16use std::sync::Arc;
17use std::time::Instant;
18
19use arc_swap::ArcSwap;
20use risingwave_backup::error::BackupError;
21use risingwave_backup::storage::{MetaSnapshotStorage, ObjectStoreMetaSnapshotStorage};
22use risingwave_backup::{MetaBackupJobId, MetaSnapshotId, MetaSnapshotManifest};
23use risingwave_common::bail;
24use risingwave_common::config::ObjectStoreConfig;
25use risingwave_common::system_param::reader::SystemParamsRead;
26use risingwave_hummock_sdk::HummockSstableObjectId;
27use risingwave_object_store::object::build_remote_object_store;
28use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
29use risingwave_pb::backup_service::{BackupJobStatus, MetaBackupManifestId};
30use risingwave_pb::meta::subscribe_response::{Info, Operation};
31use thiserror_ext::AsReport;
32use tokio::task::JoinHandle;
33
34use crate::MetaResult;
35use crate::backup_restore::meta_snapshot_builder;
36use crate::backup_restore::metrics::BackupManagerMetrics;
37use crate::hummock::sequence::next_meta_backup_id;
38use crate::hummock::{HummockManagerRef, HummockVersionSafePoint};
39use crate::manager::{LocalNotification, MetaSrvEnv};
40use crate::rpc::metrics::MetaMetrics;
41
42pub enum BackupJobResult {
43 Succeeded,
44 Failed(BackupError),
45}
46
47struct BackupJobHandle {
49 job_id: u64,
50 #[expect(dead_code)]
51 hummock_version_safe_point: HummockVersionSafePoint,
52 start_time: Instant,
53}
54
55impl BackupJobHandle {
56 pub fn new(job_id: u64, hummock_version_safe_point: HummockVersionSafePoint) -> Self {
57 Self {
58 job_id,
59 hummock_version_safe_point,
60 start_time: Instant::now(),
61 }
62 }
63}
64
65pub type BackupManagerRef = Arc<BackupManager>;
66type StoreConfig = (String, String);
68
69pub struct BackupManager {
71 env: MetaSrvEnv,
72 hummock_manager: HummockManagerRef,
73 backup_store: ArcSwap<(ObjectStoreMetaSnapshotStorage, StoreConfig)>,
74 running_job_handle: tokio::sync::Mutex<Option<BackupJobHandle>>,
76 metrics: BackupManagerMetrics,
77 meta_metrics: Arc<MetaMetrics>,
78 latest_job_info: ArcSwap<(MetaBackupJobId, BackupJobStatus, String)>,
80}
81
82impl BackupManager {
83 pub async fn new(
84 env: MetaSrvEnv,
85 hummock_manager: HummockManagerRef,
86 metrics: Arc<MetaMetrics>,
87 store_url: &str,
88 store_dir: &str,
89 ) -> MetaResult<Arc<Self>> {
90 let store_config = (store_url.to_owned(), store_dir.to_owned());
91 let store = create_snapshot_store(
92 &store_config,
93 metrics.object_store_metric.clone(),
94 &env.opts.object_store_config,
95 )
96 .await?;
97 tracing::info!(
98 "backup manager initialized: url={}, dir={}",
99 store_config.0,
100 store_config.1
101 );
102 let instance = Arc::new(Self::with_store(
103 env.clone(),
104 hummock_manager,
105 metrics,
106 (store, store_config),
107 ));
108 let (local_notification_tx, mut local_notification_rx) =
109 tokio::sync::mpsc::unbounded_channel();
110 env.notification_manager()
111 .insert_local_sender(local_notification_tx)
112 .await;
113 let this = instance.clone();
114 tokio::spawn(async move {
115 loop {
116 match local_notification_rx.recv().await {
117 Some(notification) => {
118 if let LocalNotification::SystemParamsChange(p) = notification {
119 let new_config = (
120 p.backup_storage_url().to_owned(),
121 p.backup_storage_directory().to_owned(),
122 );
123 this.handle_new_config(new_config).await;
124 }
125 }
126 None => {
127 return;
128 }
129 }
130 }
131 });
132 Ok(instance)
133 }
134
135 async fn handle_new_config(&self, new_config: StoreConfig) {
136 if self.backup_store.load().1 == new_config {
137 return;
138 }
139 if let Err(e) = self.set_store(new_config.clone()).await {
140 tracing::warn!(
142 url = &new_config.0,
143 dir = &new_config.1,
144 error = %e.as_report(),
145 "failed to apply new backup config",
146 );
147 }
148 }
149
150 fn with_store(
151 env: MetaSrvEnv,
152 hummock_manager: HummockManagerRef,
153 meta_metrics: Arc<MetaMetrics>,
154 backup_store: (ObjectStoreMetaSnapshotStorage, StoreConfig),
155 ) -> Self {
156 Self {
157 env,
158 hummock_manager,
159 backup_store: ArcSwap::from_pointee(backup_store),
160 running_job_handle: tokio::sync::Mutex::new(None),
161 metrics: BackupManagerMetrics::default(),
162 meta_metrics,
163 latest_job_info: ArcSwap::from_pointee((0, BackupJobStatus::NotFound, "".into())),
164 }
165 }
166
167 pub async fn set_store(&self, config: StoreConfig) -> MetaResult<()> {
168 let new_store = create_snapshot_store(
169 &config,
170 self.meta_metrics.object_store_metric.clone(),
171 &self.env.opts.object_store_config,
172 )
173 .await?;
174 tracing::info!(
175 "new backup config is applied: url={}, dir={}",
176 config.0,
177 config.1
178 );
179 self.backup_store.store(Arc::new((new_store, config)));
180 Ok(())
181 }
182
183 #[cfg(test)]
184 pub async fn for_test(env: MetaSrvEnv, hummock_manager: HummockManagerRef) -> Self {
185 Self::with_store(
186 env,
187 hummock_manager,
188 Arc::new(MetaMetrics::default()),
189 (
190 risingwave_backup::storage::unused().await,
191 StoreConfig::default(),
192 ),
193 )
194 }
195
196 pub async fn start_backup_job(
199 self: &Arc<Self>,
200 remarks: Option<String>,
201 ) -> MetaResult<MetaBackupJobId> {
202 let mut guard = self.running_job_handle.lock().await;
203 if let Some(job) = (*guard).as_ref() {
204 bail!(format!(
205 "concurrent backup job is not supported: existent job {}",
206 job.job_id
207 ));
208 }
209 const MAX_META_SNAPSHOT_NUM: usize = 100;
215 let current_number = self
216 .backup_store
217 .load()
218 .0
219 .manifest()
220 .snapshot_metadata
221 .len();
222 if current_number > MAX_META_SNAPSHOT_NUM {
223 bail!(format!(
224 "too many existent meta snapshots, expect at most {}",
225 MAX_META_SNAPSHOT_NUM
226 ))
227 }
228
229 let job_id = next_meta_backup_id(&self.env).await?;
230 self.latest_job_info
231 .store(Arc::new((job_id, BackupJobStatus::Running, "".into())));
232 let hummock_version_safe_point = self.hummock_manager.register_safe_point().await;
233 BackupWorker::new(self.clone()).start(job_id, remarks);
240 let job_handle = BackupJobHandle::new(job_id, hummock_version_safe_point);
241 *guard = Some(job_handle);
242 self.metrics.job_count.inc();
243 Ok(job_id)
244 }
245
246 pub fn get_backup_job_status(&self, job_id: MetaBackupJobId) -> (BackupJobStatus, String) {
247 let last = self.latest_job_info.load();
248 if last.0 == job_id {
249 return (last.1, last.2.clone());
250 }
251 (BackupJobStatus::NotFound, "".into())
252 }
253
254 async fn finish_backup_job(&self, job_id: MetaBackupJobId, job_result: BackupJobResult) {
255 let job_handle = self
257 .take_job_handle_by_job_id(job_id)
258 .await
259 .expect("job id should match");
260 let job_latency = job_handle.start_time.elapsed().as_secs_f64();
261 match job_result {
262 BackupJobResult::Succeeded => {
263 self.metrics.job_latency_success.observe(job_latency);
264 tracing::info!("succeeded backup job {}", job_id);
265 self.env
266 .notification_manager()
267 .notify_hummock_without_version(
268 Operation::Update,
269 Info::MetaBackupManifestId(MetaBackupManifestId {
270 id: self.backup_store.load().0.manifest().manifest_id,
271 }),
272 );
273 self.latest_job_info.store(Arc::new((
274 job_id,
275 BackupJobStatus::Succeeded,
276 "".into(),
277 )));
278 }
279 BackupJobResult::Failed(e) => {
280 self.metrics.job_latency_failure.observe(job_latency);
281 let message = format!("failed backup job {}: {}", job_id, e.as_report());
282 tracing::warn!(message);
283 self.latest_job_info
284 .store(Arc::new((job_id, BackupJobStatus::Failed, message)));
285 }
286 }
287 }
288
289 async fn take_job_handle_by_job_id(&self, job_id: u64) -> Option<BackupJobHandle> {
290 let mut guard = self.running_job_handle.lock().await;
291 match (*guard).as_ref() {
292 None => {
293 return None;
294 }
295 Some(job_handle) => {
296 if job_handle.job_id != job_id {
297 return None;
298 }
299 }
300 }
301 guard.take()
302 }
303
304 pub async fn delete_backups(&self, ids: &[MetaSnapshotId]) -> MetaResult<()> {
306 self.backup_store.load().0.delete(ids).await?;
307 self.env
308 .notification_manager()
309 .notify_hummock_without_version(
310 Operation::Update,
311 Info::MetaBackupManifestId(MetaBackupManifestId {
312 id: self.backup_store.load().0.manifest().manifest_id,
313 }),
314 );
315 Ok(())
316 }
317
318 pub fn list_pinned_ssts(&self) -> HashSet<HummockSstableObjectId> {
320 self.backup_store
321 .load()
322 .0
323 .manifest()
324 .snapshot_metadata
325 .iter()
326 .flat_map(|s| s.ssts.clone())
327 .collect()
328 }
329
330 pub fn manifest(&self) -> Arc<MetaSnapshotManifest> {
331 self.backup_store.load().0.manifest()
332 }
333}
334
335struct BackupWorker {
337 backup_manager: BackupManagerRef,
338}
339
340impl BackupWorker {
341 fn new(backup_manager: BackupManagerRef) -> Self {
342 Self { backup_manager }
343 }
344
345 fn start(self, job_id: u64, remarks: Option<String>) -> JoinHandle<()> {
346 let backup_manager_clone = self.backup_manager.clone();
347 let job = async move {
348 let hummock_manager = backup_manager_clone.hummock_manager.clone();
349 let hummock_version_builder = async move {
350 hummock_manager
351 .on_current_version(|version| version.clone())
352 .await
353 };
354 let meta_store = backup_manager_clone.env.meta_store();
355 let mut snapshot_builder =
356 meta_snapshot_builder::MetaSnapshotV2Builder::new(meta_store);
357 snapshot_builder
359 .build(job_id, hummock_version_builder)
360 .await?;
361 let snapshot = snapshot_builder.finish()?;
362 backup_manager_clone
363 .backup_store
364 .load()
365 .0
366 .create(&snapshot, remarks)
367 .await?;
368 Ok(BackupJobResult::Succeeded)
369 };
370 tokio::spawn(async move {
371 let job_result = job.await.unwrap_or_else(BackupJobResult::Failed);
372 self.backup_manager
373 .finish_backup_job(job_id, job_result)
374 .await;
375 })
376 }
377}
378
379async fn create_snapshot_store(
380 config: &StoreConfig,
381 metric: Arc<ObjectStoreMetrics>,
382 object_store_config: &ObjectStoreConfig,
383) -> MetaResult<ObjectStoreMetaSnapshotStorage> {
384 let object_store = Arc::new(
385 build_remote_object_store(
386 &config.0,
387 metric,
388 "Meta Backup",
389 Arc::new(object_store_config.clone()),
390 )
391 .await,
392 );
393 let store = ObjectStoreMetaSnapshotStorage::new(&config.1, object_store).await?;
394 Ok(store)
395}