risingwave_connector/source/filesystem/s3/
mod.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14pub mod enumerator;
15
16use serde::Deserialize;
17
18use crate::source::SourceProperties;
19use crate::source::filesystem::file_common::CompressionFormat;
20use crate::source::util::dummy::{
21    DummyProperties, DummySourceReader, DummySplit, DummySplitEnumerator,
22};
23
24/// Refer to [`crate::source::OPENDAL_S3_CONNECTOR`].
25pub const LEGACY_S3_CONNECTOR: &str = "s3";
26
27/// These are supported by both `s3` and `s3_v2` (opendal) sources.
28#[derive(Clone, Debug, Deserialize, PartialEq, with_options::WithOptions)]
29pub struct S3PropertiesCommon {
30    #[serde(rename = "s3.region_name")]
31    pub region_name: String,
32    #[serde(rename = "s3.bucket_name")]
33    pub bucket_name: String,
34    #[serde(rename = "match_pattern", default)]
35    pub match_pattern: Option<String>,
36    #[serde(rename = "s3.credentials.access", default)]
37    pub access: Option<String>,
38    #[serde(rename = "s3.credentials.secret", default)]
39    pub secret: Option<String>,
40    #[serde(rename = "s3.endpoint_url")]
41    pub endpoint_url: Option<String>,
42    #[serde(rename = "compression_format", default = "Default::default")]
43    pub compression_format: CompressionFormat,
44}
45
46#[derive(Debug, Clone, PartialEq)]
47pub struct LegacyS3;
48
49/// Note: legacy s3 source is fully deprecated since v2.4.0.
50/// The properties and enumerator are kept, so that meta can start normally.
51pub type LegacyS3Properties = DummyProperties<LegacyS3>;
52
53/// Note: legacy s3 source is fully deprecated since v2.4.0.
54/// The properties and enumerator are kept, so that meta can start normally.
55pub type LegacyS3SplitEnumerator = DummySplitEnumerator<LegacyS3>;
56
57pub type LegacyFsSplit = DummySplit<LegacyS3>;
58
59impl SourceProperties for LegacyS3Properties {
60    type Split = LegacyFsSplit;
61    type SplitEnumerator = LegacyS3SplitEnumerator;
62    type SplitReader = DummySourceReader<LegacyS3>;
63
64    const SOURCE_NAME: &'static str = LEGACY_S3_CONNECTOR;
65}
risingwave_connector/source/filesystem/s3/mod.rs

risingwave_connector/source/filesystem/s3/
mod.rs
