risingwave_connector/parser/maxwell/
simd_json_parser.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#[cfg(test)]
16mod tests {
17    use risingwave_common::array::Op;
18    use risingwave_common::row::Row;
19    use risingwave_common::types::{DataType, ScalarImpl, ToOwnedDatum};
20
21    use crate::parser::maxwell::MaxwellParser;
22    use crate::parser::{
23        EncodingProperties, JsonProperties, ProtocolProperties, SourceColumnDesc,
24        SourceStreamChunkBuilder, SpecificParserConfig,
25    };
26    use crate::source::{SourceContext, SourceCtrlOpts};
27
28    #[tokio::test]
29    async fn test_json_parser() {
30        let descs = vec![
31            SourceColumnDesc::simple("ID", DataType::Int32, 0.into()),
32            SourceColumnDesc::simple("NAME", DataType::Varchar, 1.into()),
33            SourceColumnDesc::simple("is_adult", DataType::Int16, 2.into()),
34            SourceColumnDesc::simple("birthday", DataType::Timestamp, 3.into()),
35        ];
36
37        let props = SpecificParserConfig {
38            encoding_config: EncodingProperties::Json(JsonProperties {
39                use_schema_registry: false,
40                timestamptz_handling: None,
41                timestamp_handling: None,
42                time_handling: None,
43            }),
44            protocol_config: ProtocolProperties::Maxwell,
45        };
46        let mut parser = MaxwellParser::new(props, descs.clone(), SourceContext::dummy().into())
47            .await
48            .unwrap();
49
50        let mut builder = SourceStreamChunkBuilder::new(descs, SourceCtrlOpts::for_test());
51        let payloads = vec![
52            br#"{"database":"test","table":"t","type":"insert","ts":1666937996,"xid":1171,"commit":true,"data":{"id":1,"name":"tom","is_adult":0,"birthday":"2017-12-31 16:00:01"}}"#.to_vec(),
53            br#"{"database":"test","table":"t","type":"insert","ts":1666938023,"xid":1254,"commit":true,"data":{"id":2,"name":"alex","is_adult":1,"birthday":"1999-12-31 16:00:01"}}"#.to_vec(),
54            br#"{"database":"test","table":"t","type":"update","ts":1666938068,"xid":1373,"commit":true,"data":{"id":2,"name":"chi","is_adult":1,"birthday":"1999-12-31 16:00:01"},"old":{"name":"alex"}}"#.to_vec()
55        ];
56
57        for payload in payloads {
58            parser
59                .parse_inner(payload, builder.row_writer())
60                .await
61                .unwrap();
62        }
63
64        builder.finish_current_chunk();
65        let chunk = builder.consume_ready_chunks().next().unwrap();
66
67        let mut rows = chunk.rows();
68
69        {
70            let (op, row) = rows.next().unwrap();
71            assert_eq!(op, Op::Insert);
72            assert_eq!(row.datum_at(0).to_owned_datum(), Some(ScalarImpl::Int32(1)));
73            assert_eq!(
74                row.datum_at(1).to_owned_datum(),
75                (Some(ScalarImpl::Utf8("tom".into())))
76            );
77            assert_eq!(
78                row.datum_at(2).to_owned_datum(),
79                (Some(ScalarImpl::Int16(0)))
80            );
81            assert_eq!(
82                row.datum_at(3).to_owned_datum(),
83                (Some(ScalarImpl::Timestamp(
84                    "2017-12-31 16:00:01".parse().unwrap()
85                )))
86            )
87        }
88
89        {
90            let (op, row) = rows.next().unwrap();
91            assert_eq!(op, Op::Insert);
92            assert_eq!(row.datum_at(0).to_owned_datum(), Some(ScalarImpl::Int32(2)));
93            assert_eq!(
94                row.datum_at(1).to_owned_datum(),
95                (Some(ScalarImpl::Utf8("alex".into())))
96            );
97            assert_eq!(
98                row.datum_at(2).to_owned_datum(),
99                (Some(ScalarImpl::Int16(1)))
100            );
101            assert_eq!(
102                row.datum_at(3).to_owned_datum(),
103                (Some(ScalarImpl::Timestamp(
104                    "1999-12-31 16:00:01".parse().unwrap()
105                )))
106            )
107        }
108
109        {
110            let (op, row) = rows.next().unwrap();
111            assert_eq!(op, Op::Insert);
112            assert_eq!(row.datum_at(0).to_owned_datum(), Some(ScalarImpl::Int32(2)));
113            assert_eq!(
114                row.datum_at(1).to_owned_datum(),
115                (Some(ScalarImpl::Utf8("chi".into())))
116            );
117            assert_eq!(
118                row.datum_at(2).to_owned_datum(),
119                (Some(ScalarImpl::Int16(1)))
120            );
121            assert_eq!(
122                row.datum_at(3).to_owned_datum(),
123                (Some(ScalarImpl::Timestamp(
124                    "1999-12-31 16:00:01".parse().unwrap()
125                )))
126            )
127        }
128    }
129}
risingwave_connector/parser/maxwell/simd_json_parser.rs

risingwave_connector/parser/maxwell/
simd_json_parser.rs
