risingwave_object_store/object/opendal_engine/
s3.rs

1// Copyright 2026 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use opendal::layers::LoggingLayer;
19use opendal::services::S3;
20use opendal::{HttpTransporter, OperationContext, Operator};
21use opendal_http_transport_reqwest::ReqwestTransport;
22use risingwave_common::config::ObjectStoreConfig;
23use thiserror_ext::AsReport as _;
24
25use super::{MediaType, OpendalObjectStore, new_operator};
26use crate::object::object_metrics::ObjectStoreMetrics;
27use crate::object::{ObjectError, ObjectResult};
28
29impl OpendalObjectStore {
30    /// create opendal s3 engine.
31    pub fn new_s3_engine(
32        bucket: String,
33        config: Arc<ObjectStoreConfig>,
34        metrics: Arc<ObjectStoreMetrics>,
35    ) -> ObjectResult<Self> {
36        // Create s3 builder.
37        let mut builder = S3::default().bucket(&bucket);
38        // For AWS S3, there is no need to set an endpoint; for other S3 compatible object stores, it is necessary to set this field.
39        if let Ok(endpoint_url) = std::env::var("RW_S3_ENDPOINT") {
40            builder = builder.endpoint(&endpoint_url);
41        }
42
43        if std::env::var("RW_IS_FORCE_PATH_STYLE").is_err() {
44            builder = builder.enable_virtual_host_style();
45        }
46
47        let http_client = Self::new_http_client(&config)?;
48        let transport = HttpTransporter::new(ReqwestTransport::new(http_client));
49
50        let op = new_operator(
51            &config,
52            Operator::new(builder)?
53                .with_context(OperationContext::new().with_http_transport(transport))
54                .layer(LoggingLayer::default()),
55        );
56
57        Ok(Self {
58            op,
59            media_type: MediaType::S3,
60            config,
61            metrics,
62        })
63    }
64
65    /// Creates a minio client. The server should be like `minio://key:secret@address:port/bucket`.
66    pub fn new_minio_engine(
67        server: &str,
68        config: Arc<ObjectStoreConfig>,
69        metrics: Arc<ObjectStoreMetrics>,
70    ) -> ObjectResult<Self> {
71        let server = server.strip_prefix("minio://").unwrap();
72        let (access_key_id, rest) = server.split_once(':').unwrap();
73        let (secret_access_key, mut rest) = rest.split_once('@').unwrap();
74
75        let endpoint_prefix = if let Some(rest_stripped) = rest.strip_prefix("https://") {
76            rest = rest_stripped;
77            "https://"
78        } else if let Some(rest_stripped) = rest.strip_prefix("http://") {
79            rest = rest_stripped;
80            "http://"
81        } else {
82            "http://"
83        };
84        let (address, bucket) = rest.split_once('/').unwrap();
85        let builder = S3::default()
86            .bucket(bucket)
87            .region("custom")
88            .access_key_id(access_key_id)
89            .secret_access_key(secret_access_key)
90            .endpoint(&format!("{}{}", endpoint_prefix, address))
91            .disable_config_load();
92
93        let http_client = Self::new_http_client(&config)?;
94        let transport = HttpTransporter::new(ReqwestTransport::new(http_client));
95
96        let op = new_operator(
97            &config,
98            Operator::new(builder)?
99                .with_context(OperationContext::new().with_http_transport(transport))
100                .layer(LoggingLayer::default()),
101        );
102
103        Ok(Self {
104            op,
105            media_type: MediaType::Minio,
106            config,
107            metrics,
108        })
109    }
110
111    pub fn new_http_client(config: &ObjectStoreConfig) -> ObjectResult<reqwest::Client> {
112        let mut client_builder = reqwest::ClientBuilder::new();
113
114        if let Some(keepalive_ms) = config.s3.keepalive_ms.as_ref() {
115            client_builder = client_builder.tcp_keepalive(Duration::from_millis(*keepalive_ms));
116        }
117
118        if let Some(nodelay) = config.s3.nodelay.as_ref() {
119            client_builder = client_builder.tcp_nodelay(*nodelay);
120        }
121        client_builder.build().map_err(|e| {
122            ObjectError::internal(format!("failed to build HTTP client: {}", e.as_report()))
123        })
124    }
125}
risingwave_object_store/object/opendal_engine/s3.rs

risingwave_object_store/object/opendal_engine/
s3.rs
