risingwave_connector/source/kinesis/
mod.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod enumerator;
16pub use enumerator::client::KinesisSplitEnumerator;
17pub mod source;
18pub mod split;
19
20use std::collections::HashMap;
21
22use serde::Deserialize;
23use serde_with::{DisplayFromStr, serde_as};
24pub use source::KinesisMeta;
25use with_options::WithOptions;
26
27use crate::connector_common::KinesisCommon;
28use crate::source::SourceProperties;
29use crate::source::kinesis::source::reader::KinesisSplitReader;
30use crate::source::kinesis::split::KinesisSplit;
31
32pub const KINESIS_CONNECTOR: &str = "kinesis";
33
34#[serde_as]
35#[derive(Clone, Debug, Deserialize, WithOptions)]
36pub struct KinesisProperties {
37    #[serde(rename = "scan.startup.mode", alias = "kinesis.scan.startup.mode")]
38    // accepted values: "latest", "earliest", "timestamp"
39    pub scan_startup_mode: Option<String>,
40
41    #[serde(rename = "scan.startup.timestamp.millis")]
42    #[serde_as(as = "Option<DisplayFromStr>")]
43    pub start_timestamp_millis: Option<i64>,
44
45    #[serde(flatten)]
46    pub common: KinesisCommon,
47
48    #[serde(flatten)]
49    pub unknown_fields: HashMap<String, String>,
50}
51
52impl SourceProperties for KinesisProperties {
53    type Split = KinesisSplit;
54    type SplitEnumerator = KinesisSplitEnumerator;
55    type SplitReader = KinesisSplitReader;
56
57    const SOURCE_NAME: &'static str = KINESIS_CONNECTOR;
58}
59
60impl crate::source::UnknownFields for KinesisProperties {
61    fn unknown_fields(&self) -> HashMap<String, String> {
62        self.unknown_fields.clone()
63    }
64}
65
66#[cfg(test)]
67mod test {
68    use maplit::hashmap;
69
70    use super::*;
71
72    #[test]
73    fn test_parse_kinesis_timestamp_offset() {
74        let props: HashMap<String, String> = hashmap! {
75            "stream".to_owned() => "sample_stream".to_owned(),
76            "aws.region".to_owned() => "us-east-1".to_owned(),
77            "scan_startup_mode".to_owned() => "timestamp".to_owned(),
78            "scan.startup.timestamp.millis".to_owned() => "123456789".to_owned(),
79        };
80
81        let kinesis_props: KinesisProperties =
82            serde_json::from_value(serde_json::to_value(props).unwrap()).unwrap();
83        assert_eq!(kinesis_props.start_timestamp_millis, Some(123456789));
84    }
85}
risingwave_connector/source/kinesis/mod.rs

risingwave_connector/source/kinesis/
mod.rs
