risingwave_connector/source/kinesis/
mod.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod enumerator;
16pub use enumerator::client::KinesisSplitEnumerator;
17pub mod source;
18pub mod split;
19
20use std::collections::HashMap;
21
22use serde::Deserialize;
23use serde_with::{DisplayFromStr, serde_as};
24pub use source::KinesisMeta;
25use with_options::WithOptions;
26
27use crate::connector_common::KinesisCommon;
28use crate::enforce_secret::EnforceSecret;
29use crate::error::ConnectorError;
30use crate::source::SourceProperties;
31use crate::source::kinesis::source::reader::KinesisSplitReader;
32use crate::source::kinesis::split::KinesisSplit;
33
34pub const KINESIS_CONNECTOR: &str = "kinesis";
35
36#[serde_as]
37#[derive(Clone, Debug, Deserialize, WithOptions)]
38pub struct KinesisProperties {
39    #[serde(rename = "scan.startup.mode", alias = "kinesis.scan.startup.mode")]
40    // accepted values: "latest", "earliest", "timestamp"
41    pub scan_startup_mode: Option<String>,
42
43    #[serde(rename = "scan.startup.timestamp.millis")]
44    #[serde_as(as = "Option<DisplayFromStr>")]
45    pub start_timestamp_millis: Option<i64>,
46
47    #[serde(flatten)]
48    pub common: KinesisCommon,
49
50    #[serde(flatten)]
51    pub reader_config: KinesisReaderConfig,
52
53    #[serde(flatten)]
54    pub unknown_fields: HashMap<String, String>,
55}
56
57const fn kinesis_reader_default_eof_retry_interval_ms() -> u64 {
58    1000
59}
60
61const fn kinesis_reader_default_error_retry_interval_ms() -> u64 {
62    200
63}
64
65#[serde_as]
66#[derive(Clone, Debug, Deserialize, WithOptions)]
67pub struct KinesisReaderConfig {
68    #[serde(
69        rename = "kinesis.reader.eof_retry_interval_ms",
70        default = "kinesis_reader_default_eof_retry_interval_ms"
71    )]
72    #[serde_as(as = "DisplayFromStr")]
73    pub eof_retry_interval_ms: u64,
74
75    #[serde(
76        rename = "kinesis.reader.error_retry_interval_ms",
77        default = "kinesis_reader_default_error_retry_interval_ms"
78    )]
79    #[serde_as(as = "DisplayFromStr")]
80    pub error_retry_interval_ms: u64,
81}
82
83impl Default for KinesisReaderConfig {
84    fn default() -> Self {
85        Self {
86            eof_retry_interval_ms: kinesis_reader_default_eof_retry_interval_ms(),
87            error_retry_interval_ms: kinesis_reader_default_error_retry_interval_ms(),
88        }
89    }
90}
91impl SourceProperties for KinesisProperties {
92    type Split = KinesisSplit;
93    type SplitEnumerator = KinesisSplitEnumerator;
94    type SplitReader = KinesisSplitReader;
95
96    const SOURCE_NAME: &'static str = KINESIS_CONNECTOR;
97}
98
99impl EnforceSecret for KinesisProperties {
100    fn enforce_secret<'a>(prop_iter: impl Iterator<Item = &'a str>) -> Result<(), ConnectorError> {
101        for prop in prop_iter {
102            KinesisCommon::enforce_one(prop)?;
103        }
104        Ok(())
105    }
106}
107
108impl crate::source::UnknownFields for KinesisProperties {
109    fn unknown_fields(&self) -> HashMap<String, String> {
110        self.unknown_fields.clone()
111    }
112}
113
114#[cfg(test)]
115mod test {
116    use maplit::{btreemap, hashmap};
117
118    use super::*;
119    use crate::WithOptionsSecResolved;
120    use crate::source::ConnectorProperties;
121
122    #[test]
123    fn test_parse_kinesis_timestamp_offset() {
124        let props: HashMap<String, String> = hashmap! {
125            "stream".to_owned() => "sample_stream".to_owned(),
126            "aws.region".to_owned() => "us-east-1".to_owned(),
127            "scan_startup_mode".to_owned() => "timestamp".to_owned(),
128            "scan.startup.timestamp.millis".to_owned() => "123456789".to_owned(),
129        };
130
131        let kinesis_props: KinesisProperties =
132            serde_json::from_value(serde_json::to_value(props).unwrap()).unwrap();
133        assert_eq!(kinesis_props.start_timestamp_millis, Some(123456789));
134    }
135
136    #[test]
137    fn test_kinesis_props() {
138        let with_options = btreemap! {
139            "connector".to_owned() => "kinesis".to_owned(),
140            "scan.startup.mode".to_owned() => "latest".to_owned(),
141            "stream".to_owned() => "sample_stream".to_owned(),
142            "aws.region".to_owned() => "us-east-1".to_owned(),
143            "endpoint".to_owned() => "https://kinesis.us-east-1.amazonaws.com".to_owned(),
144            "aws.credentials.access_key_id".to_owned() => "access_key".to_owned(),
145            "aws.credentials.secret_access_key".to_owned() => "secret_key".to_owned(),
146        };
147        let connector_props = ConnectorProperties::extract(
148            WithOptionsSecResolved::without_secrets(with_options),
149            true,
150        )
151        .unwrap();
152        let ConnectorProperties::Kinesis(kinesis_props) = connector_props else {
153            panic!()
154        };
155
156        assert_eq!(kinesis_props.common.sdk_connect_timeout_ms, 10000);
157        assert_eq!(kinesis_props.reader_config.eof_retry_interval_ms, 1000);
158        assert_eq!(kinesis_props.reader_config.error_retry_interval_ms, 200);
159
160        let with_options = btreemap! {
161            "connector".to_owned() => "kinesis".to_owned(),
162            "scan.startup.mode".to_owned() => "latest".to_owned(),
163            "stream".to_owned() => "sample_stream".to_owned(),
164            "aws.region".to_owned() => "us-east-1".to_owned(),
165            "endpoint".to_owned() => "https://kinesis.us-east-1.amazonaws.com".to_owned(),
166            "aws.credentials.access_key_id".to_owned() => "access_key".to_owned(),
167            "aws.credentials.secret_access_key".to_owned() => "secret_key".to_owned(),
168            "kinesis.sdk.connect_timeout_ms".to_owned() => "20000".to_owned(),
169            "kinesis.reader.eof_retry_interval_ms".to_owned() => "5000".to_owned(),
170            "kinesis.reader.error_retry_interval_ms".to_owned() => "1000".to_owned(),
171        };
172        let connector_props = ConnectorProperties::extract(
173            WithOptionsSecResolved::without_secrets(with_options),
174            true,
175        )
176        .unwrap();
177        let ConnectorProperties::Kinesis(kinesis_props) = connector_props else {
178            panic!()
179        };
180
181        assert_eq!(kinesis_props.common.sdk_connect_timeout_ms, 20000);
182        assert_eq!(kinesis_props.reader_config.eof_retry_interval_ms, 5000);
183        assert_eq!(kinesis_props.reader_config.error_retry_interval_ms, 1000);
184    }
185}
risingwave_connector/source/kinesis/mod.rs

risingwave_connector/source/kinesis/
mod.rs
