1use risingwave_connector::error::ConnectorError;
16use risingwave_connector::source::{ADBC_SNOWFLAKE_CONNECTOR, BATCH_POSIX_FS_CONNECTOR};
17use risingwave_pb::secret::PbSecretRef;
18
19use super::*;
20
21pub static SOURCE_ALLOWED_CONNECTION_CONNECTOR: LazyLock<HashSet<PbConnectionType>> =
22 LazyLock::new(|| {
23 hashset! {
24 PbConnectionType::Unspecified,
25 PbConnectionType::Kafka,
26 PbConnectionType::Iceberg,
27 }
28 });
29
30pub static SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY: LazyLock<HashSet<PbConnectionType>> =
31 LazyLock::new(|| {
32 hashset! {
33 PbConnectionType::Unspecified,
34 PbConnectionType::SchemaRegistry,
35 }
36 });
37
38static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, Vec<Encode>>>> =
40 LazyLock::new(|| {
41 convert_args!(hashmap!(
42 KAFKA_CONNECTOR => hashmap!(
43 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes, Encode::Csv],
44 Format::Upsert => vec![Encode::Json, Encode::Avro, Encode::Protobuf],
45 Format::Debezium => vec![Encode::Json, Encode::Avro],
46 Format::DebeziumMongo => vec![Encode::Json],
47 ),
48 PULSAR_CONNECTOR => hashmap!(
49 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes],
50 Format::Upsert => vec![Encode::Json, Encode::Avro],
51 Format::Debezium => vec![Encode::Json],
52 ),
53 KINESIS_CONNECTOR => hashmap!(
54 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes, Encode::Csv],
55 Format::Upsert => vec![Encode::Json, Encode::Avro],
56 Format::Debezium => vec![Encode::Json],
57 ),
58 GOOGLE_PUBSUB_CONNECTOR => hashmap!(
59 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes],
60 Format::Debezium => vec![Encode::Json],
61 ),
62 NEXMARK_CONNECTOR => hashmap!(
63 Format::Native => vec![Encode::Native],
64 Format::Plain => vec![Encode::Bytes],
65 ),
66 DATAGEN_CONNECTOR => hashmap!(
67 Format::Native => vec![Encode::Native],
68 Format::Plain => vec![Encode::Bytes, Encode::Json],
69 ),
70 OPENDAL_S3_CONNECTOR => hashmap!(
71 Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
72 ),
73 GCS_CONNECTOR => hashmap!(
74 Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
75 ),
76 AZBLOB_CONNECTOR => hashmap!(
77 Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
78 ),
79 POSIX_FS_CONNECTOR => hashmap!(
80 Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
81 ),
82 BATCH_POSIX_FS_CONNECTOR => hashmap!(
83 Format::Plain => vec![Encode::Csv],
84 ),
85 NATS_CONNECTOR => hashmap!(
86 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Bytes],
87 ),
88 MQTT_CONNECTOR => hashmap!(
89 Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Bytes],
90 ),
91 TEST_CONNECTOR => hashmap!(
92 Format::Plain => vec![Encode::Json],
93 ),
94 ICEBERG_CONNECTOR => hashmap!(
95 Format::None => vec![Encode::None],
96 ),
97 ADBC_SNOWFLAKE_CONNECTOR => hashmap!(
98 Format::None => vec![Encode::None],
99 ),
100 ))
101 });
102
103fn validate_pulsar_schema_options(
107 format_encode: &FormatEncodeOptions,
108 connector: &str,
109) -> Result<bool> {
110 let options = WithOptions::try_from(format_encode.row_options())?;
111 if !uses_pulsar_schema(&options, options.secret_ref()) {
112 return Ok(false);
113 }
114
115 if connector != PULSAR_CONNECTOR
116 || format_encode.format != Format::Plain
117 || !matches!(format_encode.row_encode, Encode::Avro | Encode::Json)
118 {
119 return Err(RwError::from(ProtocolError(format!(
120 "Pulsar schema requires connector = '{PULSAR_CONNECTOR}' with FORMAT PLAIN ENCODE AVRO or JSON"
121 ))));
122 }
123
124 Ok(true)
125}
126
127pub fn validate_compatibility(
128 format_encode: &FormatEncodeOptions,
129 props: &mut BTreeMap<String, String>,
130 secret_refs: &BTreeMap<String, PbSecretRef>,
131) -> Result<()> {
132 let mut connector = props
133 .get_connector()
134 .ok_or_else(|| RwError::from(ProtocolError("missing field 'connector'".to_owned())))?;
135 let uses_pulsar_schema = validate_pulsar_schema_options(format_encode, &connector)?;
136
137 let protocol_error = |e: ConnectorError| RwError::from(ProtocolError(e.to_report_string()));
138 dispatch_cdc_source_type!(
139 props.cdc_source_type(),
140 T,
141 {
142 T::check_license()?;
143 T::check_format_encode(
144 format_to_prost(&format_encode.format),
145 row_encode_to_prost(&format_encode.row_encode),
146 )
147 .map_err(protocol_error)?;
148 T::check_and_fill_create_with_options(props, secret_refs).map_err(protocol_error)?;
149 return Ok(());
150 },
151 {}
152 );
153
154 if connector == OPENDAL_S3_CONNECTOR {
155 return Err(RwError::from(Deprecated(
157 OPENDAL_S3_CONNECTOR.to_owned(),
158 LEGACY_S3_CONNECTOR.to_owned(),
159 )));
160 }
161 if connector == LEGACY_S3_CONNECTOR {
162 let entry = props.get_mut(UPSTREAM_SOURCE_KEY).unwrap();
165 *entry = OPENDAL_S3_CONNECTOR.to_owned();
166 connector = OPENDAL_S3_CONNECTOR.to_owned();
167 }
168
169 let compatible_formats = CONNECTORS_COMPATIBLE_FORMATS
170 .get(&connector)
171 .ok_or_else(|| {
172 RwError::from(ProtocolError(format!(
173 "connector {:?} is not supported, accept {:?}",
174 connector,
175 CONNECTORS_COMPATIBLE_FORMATS.keys()
176 )))
177 })?;
178 if connector != KAFKA_CONNECTOR && !uses_pulsar_schema {
179 let res = match (&format_encode.format, &format_encode.row_encode) {
180 (Format::Plain | Format::Upsert, Encode::Protobuf | Encode::Avro) => {
181 let mut options = WithOptions::try_from(format_encode.row_options())?;
182 let (_, use_schema_registry) =
183 get_schema_location(options.inner_mut()).unwrap_or_default();
184 use_schema_registry
185 }
186 (Format::Debezium, Encode::Avro) => true,
187 (_, _) => false,
188 };
189 if res {
190 return Err(RwError::from(ProtocolError(format!(
191 "The {} must be kafka when schema registry is used",
192 UPSTREAM_SOURCE_KEY
193 ))));
194 }
195 }
196
197 let compatible_encodes = compatible_formats
198 .get(&format_encode.format)
199 .ok_or_else(|| {
200 RwError::from(ProtocolError(format!(
201 "connector {} does not support format {:?}",
202 connector, format_encode.format
203 )))
204 })?;
205 if !compatible_encodes.contains(&format_encode.row_encode) {
206 return Err(RwError::from(ProtocolError(format!(
207 "connector {} does not support format {:?} with encode {:?}",
208 connector, format_encode.format, format_encode.row_encode
209 ))));
210 }
211 Ok(())
212}
213
214#[cfg(test)]
215mod tests {
216 use risingwave_sqlparser::ast::SqlOption;
217
218 use super::*;
219
220 fn format_encode(
221 format: Format,
222 encode: Encode,
223 options: &[(&str, &str)],
224 ) -> FormatEncodeOptions {
225 let row_options = options
226 .iter()
227 .map(|(name, value)| {
228 let name = name.to_string();
229 let value = value.to_string();
230 SqlOption::try_from((&name, &value)).unwrap()
231 })
232 .collect();
233 FormatEncodeOptions {
234 format,
235 row_encode: encode,
236 row_options,
237 key_encode: None,
238 }
239 }
240
241 fn source_options(connector: &str) -> BTreeMap<String, String> {
242 BTreeMap::from([(UPSTREAM_SOURCE_KEY.to_owned(), connector.to_owned())])
243 }
244
245 #[test]
246 fn pulsar_schema_accepts_minimal_plain_avro_and_json_options() {
247 for encode in [Encode::Avro, Encode::Json] {
248 let format_encode = format_encode(
249 Format::Plain,
250 encode,
251 &[
252 (PULSAR_SCHEMA_URL_KEY, "https://localhost:8443"),
253 (PULSAR_SCHEMA_AUTH_TOKEN_KEY, "schema-token"),
254 ],
255 );
256 validate_compatibility(
257 &format_encode,
258 &mut source_options(PULSAR_CONNECTOR),
259 &BTreeMap::new(),
260 )
261 .unwrap();
262 }
263 }
264
265 #[test]
266 fn pulsar_schema_rejects_other_connectors_and_formats() {
267 for (connector, format, encode) in [
268 (KAFKA_CONNECTOR, Format::Plain, Encode::Avro),
269 (PULSAR_CONNECTOR, Format::Upsert, Encode::Avro),
270 (KAFKA_CONNECTOR, Format::Plain, Encode::Json),
271 (PULSAR_CONNECTOR, Format::Upsert, Encode::Json),
272 (PULSAR_CONNECTOR, Format::Plain, Encode::Protobuf),
273 ] {
274 let format_encode = format_encode(
275 format,
276 encode,
277 &[(PULSAR_SCHEMA_URL_KEY, "http://localhost:8080")],
278 );
279 let error = validate_compatibility(
280 &format_encode,
281 &mut source_options(connector),
282 &BTreeMap::new(),
283 )
284 .unwrap_err();
285 assert!(error.to_string().contains(
286 "Pulsar schema requires connector = 'pulsar' with FORMAT PLAIN ENCODE AVRO or JSON"
287 ));
288 }
289 }
290
291 #[test]
292 fn confluent_schema_registry_behavior_is_unchanged() {
293 let format_encode = format_encode(
294 Format::Plain,
295 Encode::Avro,
296 &[("schema.registry", "http://localhost:8081")],
297 );
298 validate_compatibility(
299 &format_encode,
300 &mut source_options(KAFKA_CONNECTOR),
301 &BTreeMap::new(),
302 )
303 .unwrap();
304 }
305
306 #[test]
307 fn glue_schema_registry_non_kafka() {
308 let format_encode = format_encode(
309 Format::Plain,
310 Encode::Avro,
311 &[(AWS_GLUE_SCHEMA_ARN_KEY, "placeholder")],
312 );
313 validate_compatibility(
314 &format_encode,
315 &mut source_options(KINESIS_CONNECTOR),
316 &BTreeMap::new(),
317 )
318 .unwrap();
319 }
320}