risingwave_object_store/object/opendal_engine/
opendal_object_store.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use bytes::{Bytes, BytesMut};
19use fail::fail_point;
20use futures::{StreamExt, stream};
21use opendal::layers::{RetryLayer, TimeoutLayer};
22use opendal::raw::BoxedStaticFuture;
23use opendal::services::Memory;
24use opendal::{Execute, Executor, Operator, Writer};
25use risingwave_common::config::ObjectStoreConfig;
26use risingwave_common::range::RangeBoundsExt;
27use thiserror_ext::AsReport;
28
29use crate::object::object_metrics::ObjectStoreMetrics;
30use crate::object::{
31    MonitoredObjectStore, ObjectDataStream, ObjectError, ObjectMetadata, ObjectMetadataIter,
32    ObjectRangeBounds, ObjectResult, ObjectStore, OperationType, StreamingUploader, prefix,
33};
34
35/// Opendal object storage.
36#[derive(Clone)]
37pub struct OpendalObjectStore {
38    pub(crate) op: Operator,
39    pub(crate) media_type: MediaType,
40    pub(crate) config: Arc<ObjectStoreConfig>,
41    pub(crate) metrics: Arc<ObjectStoreMetrics>,
42}
43
44#[derive(Clone)]
45pub enum MediaType {
46    Memory,
47    Hdfs,
48    Gcs,
49    Minio,
50    S3,
51    Obs,
52    Oss,
53    Webhdfs,
54    Azblob,
55    Fs,
56}
57
58impl MediaType {
59    pub fn as_str(&self) -> &'static str {
60        match self {
61            MediaType::Memory => "Memory",
62            MediaType::Hdfs => "Hdfs",
63            MediaType::Gcs => "Gcs",
64            MediaType::Minio => "Minio",
65            MediaType::S3 => "S3",
66            MediaType::Obs => "Obs",
67            MediaType::Oss => "Oss",
68            MediaType::Webhdfs => "Webhdfs",
69            MediaType::Azblob => "Azblob",
70            MediaType::Fs => "Fs",
71        }
72    }
73}
74
75impl OpendalObjectStore {
76    /// create opendal memory engine, used for unit tests.
77    pub fn test_new_memory_engine() -> ObjectResult<Self> {
78        // Create memory backend builder.
79        let builder = Memory::default();
80        let op: Operator = Operator::new(builder)?.finish();
81
82        Ok(Self {
83            op,
84            media_type: MediaType::Memory,
85            config: Arc::new(ObjectStoreConfig::default()),
86            metrics: Arc::new(ObjectStoreMetrics::unused()),
87        })
88    }
89}
90
91#[inline]
92fn timestamp_to_secs(t: opendal::raw::Timestamp) -> f64 {
93    t.into_inner().as_second() as f64
94}
95
96#[async_trait::async_trait]
97impl ObjectStore for OpendalObjectStore {
98    type StreamingUploader = OpendalStreamingUploader;
99
100    fn get_object_prefix(&self, obj_id: u64, use_new_object_prefix_strategy: bool) -> String {
101        match self.media_type {
102            MediaType::S3 => prefix::s3::get_object_prefix(obj_id),
103            MediaType::Minio => prefix::s3::get_object_prefix(obj_id),
104            MediaType::Memory => String::default(),
105            MediaType::Hdfs
106            | MediaType::Gcs
107            | MediaType::Obs
108            | MediaType::Oss
109            | MediaType::Webhdfs
110            | MediaType::Azblob
111            | MediaType::Fs => {
112                prefix::opendal_engine::get_object_prefix(obj_id, use_new_object_prefix_strategy)
113            }
114        }
115    }
116
117    async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()> {
118        if obj.is_empty() {
119            Err(ObjectError::internal("upload empty object"))
120        } else {
121            self.op.write(path, obj).await?;
122            Ok(())
123        }
124    }
125
126    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader> {
127        if self.op.info().native_capability().write_can_multi {
128            Ok(OpendalStreamingUploader::Native(Box::new(
129                OpendalNativeStreamingUploader::new(
130                    self.op.clone(),
131                    path.to_owned(),
132                    self.config.clone(),
133                    self.metrics.clone(),
134                    self.store_media_type(),
135                )
136                .await?,
137            )))
138        } else {
139            Ok(OpendalStreamingUploader::Buffered(
140                OpendalBufferedStreamingUploader::new(
141                    self.clone()
142                        .monitored(self.metrics.clone(), self.config.clone()),
143                    path.to_owned(),
144                ),
145            ))
146        }
147    }
148
149    async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
150        let data = if range.is_full() {
151            self.op.read(path).await?
152        } else {
153            self.op
154                .read_with(path)
155                .range(range.map(|v| *v as u64))
156                .await?
157        };
158
159        if let Some(len) = range.len()
160            && len != data.len()
161        {
162            return Err(ObjectError::internal(format!(
163                "mismatched size: expected {}, found {} when reading {} at {:?}",
164                len,
165                data.len(),
166                path,
167                range,
168            )));
169        }
170
171        Ok(data.to_bytes())
172    }
173
174    /// Returns a stream reading the object specified in `path`. If given, the stream starts at the
175    /// byte with index `start_pos` (0-based). As far as possible, the stream only loads the amount
176    /// of data into memory that is read from the stream.
177    async fn streaming_read(
178        &self,
179        path: &str,
180        range: impl ObjectRangeBounds,
181    ) -> ObjectResult<ObjectDataStream> {
182        fail_point!("opendal_streaming_read_err", |_| Err(
183            ObjectError::internal("opendal streaming read error")
184        ));
185        let range = range.map(|v| *v as u64);
186
187        // The layer specified first will be executed first.
188        // `TimeoutLayer` must be specified before `RetryLayer`.
189        // Otherwise, it will lead to bad state inside OpenDAL and panic.
190        // See https://docs.rs/opendal/latest/opendal/layers/struct.RetryLayer.html#panics
191        let reader = self
192            .op
193            .clone()
194            .layer(TimeoutLayer::new().with_io_timeout(Duration::from_millis(
195                self.config.retry.streaming_read_attempt_timeout_ms,
196            )))
197            .layer(
198                RetryLayer::new()
199                    .with_min_delay(Duration::from_millis(
200                        self.config.retry.req_backoff_interval_ms,
201                    ))
202                    .with_max_delay(Duration::from_millis(
203                        self.config.retry.req_backoff_max_delay_ms,
204                    ))
205                    .with_max_times(self.config.retry.streaming_read_retry_attempts)
206                    .with_factor(self.config.retry.req_backoff_factor as f32)
207                    .with_jitter(),
208            )
209            .reader_with(path)
210            .await?;
211        let stream = reader.into_bytes_stream(range).await?.map(|item| {
212            item.map(|b| Bytes::copy_from_slice(b.as_ref()))
213                .map_err(|e| {
214                    ObjectError::internal(format!("reader into_stream fail {}", e.as_report()))
215                })
216        });
217
218        Ok(Box::pin(stream))
219    }
220
221    async fn metadata(&self, path: &str) -> ObjectResult<ObjectMetadata> {
222        let opendal_metadata = self.op.stat(path).await?;
223        let key = path.to_owned();
224        let last_modified = match opendal_metadata.last_modified() {
225            Some(t) => timestamp_to_secs(t),
226            None => 0_f64,
227        };
228
229        let total_size = opendal_metadata.content_length() as usize;
230        let metadata = ObjectMetadata {
231            key,
232            last_modified,
233            total_size,
234        };
235        Ok(metadata)
236    }
237
238    async fn delete(&self, path: &str) -> ObjectResult<()> {
239        self.op.delete(path).await?;
240        Ok(())
241    }
242
243    /// Deletes the objects with the given paths permanently from the storage. If an object
244    /// specified in the request is not found, it will be considered as successfully deleted.
245    async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()> {
246        self.op.delete_iter(paths.to_vec()).await?;
247        Ok(())
248    }
249
250    async fn list(
251        &self,
252        prefix: &str,
253        start_after: Option<String>,
254        limit: Option<usize>,
255    ) -> ObjectResult<ObjectMetadataIter> {
256        let mut object_lister = self.op.lister_with(prefix).recursive(true);
257        if let Some(start_after) = start_after {
258            object_lister = object_lister.start_after(&start_after);
259        }
260        let object_lister = object_lister.await?;
261
262        let op = self.op.clone();
263        let stream = stream::unfold(object_lister, move |mut object_lister| {
264            let op = op.clone();
265
266            async move {
267                match object_lister.next().await {
268                    Some(Ok(object)) => {
269                        let key = object.path().to_owned();
270
271                        // OpenDAL 0.55 removed list metadata capability flags and reports
272                        // unknown content length as 0. Use listed metadata first, and call
273                        // stat() if timestamp is missing or size is 0 to avoid treating
274                        // unknown sizes as real zero-byte objects.
275                        let meta = object.metadata();
276                        let mut last_modified = meta.last_modified().map(timestamp_to_secs);
277                        let mut total_size = meta.content_length() as usize;
278                        if last_modified.is_none() || total_size == 0 {
279                            let stat_meta = op.stat(&key).await.ok()?;
280                            last_modified = stat_meta.last_modified().map(timestamp_to_secs);
281                            total_size = stat_meta.content_length() as usize;
282                        }
283
284                        let metadata = ObjectMetadata {
285                            key,
286                            last_modified: last_modified.unwrap_or(0_f64),
287                            total_size,
288                        };
289                        Some((Ok(metadata), object_lister))
290                    }
291                    Some(Err(err)) => Some((Err(err.into()), object_lister)),
292                    None => None,
293                }
294            }
295        });
296
297        Ok(stream.take(limit.unwrap_or(usize::MAX)).boxed())
298    }
299
300    fn store_media_type(&self) -> &'static str {
301        self.media_type.as_str()
302    }
303}
304
305impl OpendalObjectStore {
306    pub async fn copy(&self, from_path: &str, to_path: &str) -> ObjectResult<()> {
307        self.op.copy(from_path, to_path).await?;
308        Ok(())
309    }
310}
311
312struct OpendalStreamingUploaderExecute {
313    /// To record metrics for uploading part.
314    metrics: Arc<ObjectStoreMetrics>,
315    media_type: &'static str,
316}
317
318impl OpendalStreamingUploaderExecute {
319    const STREAMING_UPLOAD_TYPE: OperationType = OperationType::StreamingUpload;
320
321    fn new(metrics: Arc<ObjectStoreMetrics>, media_type: &'static str) -> Self {
322        Self {
323            metrics,
324            media_type,
325        }
326    }
327}
328
329impl Execute for OpendalStreamingUploaderExecute {
330    fn execute(&self, f: BoxedStaticFuture<()>) {
331        let operation_type_str = Self::STREAMING_UPLOAD_TYPE.as_str();
332        let media_type = self.media_type;
333
334        let metrics = self.metrics.clone();
335        let _handle = tokio::spawn(async move {
336            let _timer = metrics
337                .operation_latency
338                .with_label_values(&[media_type, operation_type_str])
339                .start_timer();
340
341            f.await
342        });
343    }
344}
345
346pub enum OpendalStreamingUploader {
347    Native(Box<OpendalNativeStreamingUploader>),
348    Buffered(OpendalBufferedStreamingUploader),
349}
350
351/// Store multiple parts in a map, and concatenate them on finish.
352pub struct OpendalNativeStreamingUploader {
353    writer: Writer,
354    /// Buffer for data. It will store at least `UPLOAD_BUFFER_SIZE` bytes of data before wrapping itself
355    /// into a stream and upload to object store as a part.
356    buf: Vec<Bytes>,
357    /// Length of the data that have not been uploaded to object store.
358    not_uploaded_len: usize,
359    /// Whether the writer is valid. The writer is invalid after abort/close.
360    is_valid: bool,
361
362    abort_on_err: bool,
363
364    upload_part_size: usize,
365}
366
367impl OpendalNativeStreamingUploader {
368    pub async fn new(
369        op: Operator,
370        path: String,
371        config: Arc<ObjectStoreConfig>,
372        metrics: Arc<ObjectStoreMetrics>,
373        media_type: &'static str,
374    ) -> ObjectResult<Self> {
375        let monitored_execute = OpendalStreamingUploaderExecute::new(metrics, media_type);
376        let executor = Executor::with(monitored_execute);
377        op.update_executor(|_| executor);
378
379        let writer = op
380            .clone()
381            .layer(TimeoutLayer::new().with_io_timeout(Duration::from_millis(
382                config.retry.streaming_upload_attempt_timeout_ms,
383            )))
384            .layer(
385                RetryLayer::new()
386                    .with_min_delay(Duration::from_millis(config.retry.req_backoff_interval_ms))
387                    .with_max_delay(Duration::from_millis(config.retry.req_backoff_max_delay_ms))
388                    .with_max_times(config.retry.streaming_upload_retry_attempts)
389                    .with_factor(config.retry.req_backoff_factor as f32)
390                    .with_jitter(),
391            )
392            .writer_with(&path)
393            .concurrent(config.opendal_upload_concurrency)
394            .await?;
395        Ok(Self {
396            writer,
397            buf: vec![],
398            not_uploaded_len: 0,
399            is_valid: true,
400            abort_on_err: config.opendal_writer_abort_on_err,
401            upload_part_size: config.upload_part_size,
402        })
403    }
404
405    async fn flush(&mut self) -> ObjectResult<()> {
406        let data: Vec<Bytes> = self.buf.drain(..).collect();
407        debug_assert_eq!(
408            data.iter().map(|b| b.len()).sum::<usize>(),
409            self.not_uploaded_len
410        );
411        if let Err(err) = self.writer.write(data).await {
412            self.is_valid = false;
413            if self.abort_on_err {
414                self.writer.abort().await?;
415            }
416            return Err(err.into());
417        }
418        self.not_uploaded_len = 0;
419        Ok(())
420    }
421}
422
423impl StreamingUploader for OpendalNativeStreamingUploader {
424    async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
425        assert!(self.is_valid);
426        self.not_uploaded_len += data.len();
427        self.buf.push(data);
428        if self.not_uploaded_len >= self.upload_part_size {
429            self.flush().await?;
430        }
431        Ok(())
432    }
433
434    async fn finish(mut self) -> ObjectResult<()> {
435        assert!(self.is_valid);
436        if self.not_uploaded_len > 0 {
437            self.flush().await?;
438        }
439
440        assert!(self.buf.is_empty());
441        assert_eq!(self.not_uploaded_len, 0);
442
443        self.is_valid = false;
444        match self.writer.close().await {
445            Ok(_) => (),
446            Err(err) => {
447                if self.abort_on_err {
448                    self.writer.abort().await?;
449                }
450                return Err(err.into());
451            }
452        };
453
454        Ok(())
455    }
456
457    // Not absolutely accurate. Some bytes may be in the infight request.
458    fn get_memory_usage(&self) -> u64 {
459        self.not_uploaded_len as u64
460    }
461}
462
463pub struct OpendalBufferedStreamingUploader {
464    store: MonitoredObjectStore<OpendalObjectStore>,
465    path: String,
466    buf: BytesMut,
467}
468
469impl OpendalBufferedStreamingUploader {
470    fn new(store: MonitoredObjectStore<OpendalObjectStore>, path: String) -> Self {
471        Self {
472            store,
473            path,
474            buf: BytesMut::new(),
475        }
476    }
477}
478
479impl StreamingUploader for OpendalBufferedStreamingUploader {
480    async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
481        self.buf.extend_from_slice(&data);
482        Ok(())
483    }
484
485    async fn finish(self) -> ObjectResult<()> {
486        self.store.upload(&self.path, self.buf.freeze()).await
487    }
488
489    fn get_memory_usage(&self) -> u64 {
490        self.buf.len() as u64
491    }
492}
493
494impl StreamingUploader for OpendalStreamingUploader {
495    async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
496        match self {
497            Self::Native(uploader) => uploader.write_bytes(data).await,
498            Self::Buffered(uploader) => uploader.write_bytes(data).await,
499        }
500    }
501
502    async fn finish(self) -> ObjectResult<()> {
503        match self {
504            Self::Native(uploader) => (*uploader).finish().await,
505            Self::Buffered(uploader) => uploader.finish().await,
506        }
507    }
508
509    fn get_memory_usage(&self) -> u64 {
510        match self {
511            Self::Native(uploader) => uploader.get_memory_usage(),
512            Self::Buffered(uploader) => uploader.get_memory_usage(),
513        }
514    }
515}
516
517#[cfg(test)]
518mod tests {
519    use stream::TryStreamExt;
520
521    use super::*;
522
523    async fn list_all(prefix: &str, store: &OpendalObjectStore) -> Vec<ObjectMetadata> {
524        store
525            .list(prefix, None, None)
526            .await
527            .unwrap()
528            .try_collect::<Vec<_>>()
529            .await
530            .unwrap()
531    }
532
533    #[tokio::test]
534    async fn test_memory_upload() {
535        let block = Bytes::from("123456");
536        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
537        store.upload("/abc", block).await.unwrap();
538
539        // No such object.
540        store.read("/ab", 0..3).await.unwrap_err();
541
542        let bytes = store.read("/abc", 4..6).await.unwrap();
543        assert_eq!(String::from_utf8(bytes.to_vec()).unwrap(), "56".to_owned());
544
545        store.delete("/abc").await.unwrap();
546
547        // No such object.
548        store.read("/abc", 0..3).await.unwrap_err();
549    }
550
551    #[tokio::test]
552    #[should_panic]
553    async fn test_memory_read_overflow() {
554        let block = Bytes::from("123456");
555        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
556        store.upload("/abc", block).await.unwrap();
557
558        // Overflow.
559        store.read("/abc", 4..44).await.unwrap_err();
560    }
561
562    #[tokio::test]
563    async fn test_memory_metadata() {
564        let block = Bytes::from("123456");
565        let path = "/abc".to_owned();
566        let obj_store = OpendalObjectStore::test_new_memory_engine().unwrap();
567        obj_store.upload("/abc", block).await.unwrap();
568
569        let err = obj_store.metadata("/not_exist").await.unwrap_err();
570        assert!(err.is_object_not_found_error());
571        let metadata = obj_store.metadata("/abc").await.unwrap();
572        assert_eq!(metadata.total_size, 6);
573        obj_store.delete(&path).await.unwrap();
574    }
575
576    #[tokio::test]
577    async fn test_buffered_streaming_upload() {
578        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
579        let mut uploader = OpendalBufferedStreamingUploader::new(
580            store
581                .clone()
582                .monitored(store.metrics.clone(), store.config.clone()),
583            "/abc".to_owned(),
584        );
585
586        uploader.write_bytes(Bytes::from("123")).await.unwrap();
587        assert_eq!(uploader.get_memory_usage(), 3);
588        uploader.write_bytes(Bytes::from("456")).await.unwrap();
589        assert_eq!(uploader.get_memory_usage(), 6);
590        uploader.finish().await.unwrap();
591
592        let read_obj = store.read("/abc", ..).await.unwrap();
593        assert_eq!(read_obj, Bytes::from("123456"));
594    }
595
596    #[tokio::test]
597    async fn test_empty_buffered_streaming_upload() {
598        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
599        let uploader = OpendalBufferedStreamingUploader::new(
600            store
601                .clone()
602                .monitored(store.metrics.clone(), store.config.clone()),
603            "/abc".to_owned(),
604        );
605
606        uploader.finish().await.unwrap_err();
607    }
608
609    #[tokio::test]
610    async fn test_memory_delete_objects_and_list_object() {
611        let block1 = Bytes::from("123456");
612        let block2 = Bytes::from("987654");
613        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
614        store.upload("abc", Bytes::from("123456")).await.unwrap();
615        store.upload("prefix/abc", block1).await.unwrap();
616
617        store.upload("prefix/xyz", block2).await.unwrap();
618
619        assert_eq!(list_all("", &store).await.len(), 3);
620        assert_eq!(list_all("prefix/", &store).await.len(), 2);
621        let str_list = [String::from("prefix/abc"), String::from("prefix/xyz")];
622
623        store.delete_objects(&str_list).await.unwrap();
624
625        assert!(store.read("prefix/abc/", ..).await.is_err());
626        assert!(store.read("prefix/xyz/", ..).await.is_err());
627        assert_eq!(list_all("", &store).await.len(), 1);
628        assert_eq!(list_all("prefix/", &store).await.len(), 0);
629    }
630}
risingwave_object_store/object/opendal_engine/opendal_object_store.rs

risingwave_object_store/object/opendal_engine/
opendal_object_store.rs
