1use risingwave_connector::source::ADBC_SNOWFLAKE_CONNECTOR;
18
19use super::*;
20
21mod json;
22use json::*;
23mod avro;
24use avro::extract_avro_table_schema;
25pub mod debezium;
26pub mod iceberg;
27use iceberg::extract_iceberg_columns;
28mod protobuf;
29use protobuf::extract_protobuf_table_schema;
30pub mod nexmark;
31
32macro_rules! feature_gated_function {
37 (
38 mod $mod_name:ident,
39 $feature_name:literal,
40 async fn $func_name:ident ( $( $param_name:ident : $param_type:ty ),* $(,)? ) -> $ret_type:ty
41 ) => {
42 #[cfg(feature = $feature_name)]
43 pub mod $mod_name;
44 #[cfg(feature = $feature_name)]
45 use $mod_name::$func_name;
46
47 #[cfg(not(feature = $feature_name))]
48 #[allow(unused_variables)]
49 pub async fn $func_name( $( $param_name : $param_type ),* ) -> $ret_type {
50 Err(anyhow::anyhow!(
51 "Feature `{}` is not enabled at compile time. \
52 Please enable it in `Cargo.toml` and rebuild.",
53 $feature_name
54 ))
55 }
56 };
57}
58
59feature_gated_function!(
60 mod adbc_snowflake,
61 "source-adbc_snowflake",
62 async fn extract_adbc_snowflake_columns(
63 with_properties: &WithOptionsSecResolved,
64 ) -> anyhow::Result<Vec<ColumnCatalog>>
65);
66
67pub async fn bind_columns_from_source(
73 session: &SessionImpl,
74 format_encode: &FormatEncodeOptions,
75 with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
76 create_source_type: CreateSourceType,
77) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
78 let (columns_from_resolve_source, mut source_info) =
79 if create_source_type == CreateSourceType::SharedCdc {
80 bind_columns_from_source_for_cdc(session, format_encode)?
81 } else {
82 bind_columns_from_source_for_non_cdc(session, format_encode, with_properties).await?
83 };
84 if create_source_type.is_shared() {
85 source_info.cdc_source_job = true;
87 source_info.is_distributed = create_source_type == CreateSourceType::SharedNonCdc;
88 }
89 Ok((columns_from_resolve_source, source_info))
90}
91
92async fn bind_columns_from_source_for_non_cdc(
93 session: &SessionImpl,
94 format_encode: &FormatEncodeOptions,
95 with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
96) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
97 const MESSAGE_NAME_KEY: &str = "message";
98 const KEY_MESSAGE_NAME_KEY: &str = "key.message";
99 const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy";
100
101 let options_with_secret = match with_properties {
102 Either::Left(options) => {
103 let (sec_resolve_props, connection_type, _) = resolve_connection_ref_and_secret_ref(
104 options.clone(),
105 session,
106 Some(TelemetryDatabaseObject::Source),
107 )?;
108 if !SOURCE_ALLOWED_CONNECTION_CONNECTOR.contains(&connection_type) {
109 return Err(RwError::from(ProtocolError(format!(
110 "connection type {:?} is not allowed, allowed types: {:?}",
111 connection_type, SOURCE_ALLOWED_CONNECTION_CONNECTOR
112 ))));
113 }
114
115 sec_resolve_props
116 }
117 Either::Right(options_with_secret) => options_with_secret.clone(),
118 };
119
120 let is_kafka: bool = options_with_secret.is_kafka_connector();
121
122 let (sec_resolve_props, connection_type, schema_registry_conn_ref) =
124 resolve_connection_ref_and_secret_ref(
125 WithOptions::try_from(format_encode.row_options())?,
126 session,
127 Some(TelemetryDatabaseObject::Source),
128 )?;
129 ensure_connection_type_allowed(connection_type, &SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY)?;
130
131 let (format_encode_options, format_encode_secret_refs) = sec_resolve_props.into_parts();
132 let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
134 format_encode_options.clone(),
135 format_encode_secret_refs.clone(),
136 )?;
137
138 fn get_key_message_name(options: &mut BTreeMap<String, String>) -> Option<String> {
139 consume_string_from_options(options, KEY_MESSAGE_NAME_KEY).ok()
140 }
141 fn get_sr_name_strategy_check(
142 options: &mut BTreeMap<String, String>,
143 use_sr: bool,
144 ) -> Result<Option<i32>> {
145 let name_strategy = get_name_strategy_or_default(try_consume_string_from_options(
146 options,
147 NAME_STRATEGY_KEY,
148 ))?;
149 if !use_sr && name_strategy.is_some() {
150 return Err(RwError::from(ProtocolError(
151 "schema registry name strategy only works with schema registry enabled".to_owned(),
152 )));
153 }
154 Ok(name_strategy)
155 }
156
157 let mut stream_source_info = StreamSourceInfo {
158 format: format_to_prost(&format_encode.format) as i32,
159 row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
160 format_encode_options,
161 format_encode_secret_refs,
162 connection_id: schema_registry_conn_ref,
163 ..Default::default()
164 };
165
166 if format_encode.format == Format::Debezium {
167 try_consume_string_from_options(&mut format_encode_options_to_consume, DEBEZIUM_IGNORE_KEY);
168 }
169
170 let columns = match (&format_encode.format, &format_encode.row_encode) {
171 (Format::Native, Encode::Native)
172 | (Format::Plain, Encode::Bytes)
173 | (Format::DebeziumMongo, Encode::Json) => None,
174 (Format::Plain, Encode::Protobuf) | (Format::Upsert, Encode::Protobuf) => {
175 let (row_schema_location, use_schema_registry) =
176 get_schema_location(&mut format_encode_options_to_consume)?;
177 let message_name = consume_string_from_options(
178 &mut format_encode_options_to_consume,
179 MESSAGE_NAME_KEY,
180 )?;
181 let name_strategy = get_sr_name_strategy_check(
182 &mut format_encode_options_to_consume,
183 use_schema_registry,
184 )?;
185
186 stream_source_info.use_schema_registry = use_schema_registry;
187 stream_source_info
188 .row_schema_location
189 .clone_from(&row_schema_location);
190 stream_source_info
191 .proto_message_name
192 .clone_from(&message_name);
193 stream_source_info.key_message_name =
194 get_key_message_name(&mut format_encode_options_to_consume);
195 stream_source_info.name_strategy =
196 name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
197
198 Some(
199 extract_protobuf_table_schema(
200 &stream_source_info,
201 &options_with_secret,
202 &mut format_encode_options_to_consume,
203 )
204 .await?,
205 )
206 }
207 (format @ (Format::Plain | Format::Upsert | Format::Debezium), Encode::Avro) => {
208 if format_encode_options_to_consume.contains_key(PULSAR_SCHEMA_URL_KEY) {
209 } else if format_encode_options_to_consume
213 .remove(AWS_GLUE_SCHEMA_ARN_KEY)
214 .is_none()
215 {
216 let (row_schema_location, use_schema_registry) =
222 get_schema_location(&mut format_encode_options_to_consume)?;
223
224 if matches!(format, Format::Debezium) && !use_schema_registry {
225 return Err(RwError::from(ProtocolError(
226 "schema location for DEBEZIUM_AVRO row format is not supported".to_owned(),
227 )));
228 }
229
230 let message_name = try_consume_string_from_options(
231 &mut format_encode_options_to_consume,
232 MESSAGE_NAME_KEY,
233 );
234 let name_strategy = get_sr_name_strategy_check(
235 &mut format_encode_options_to_consume,
236 use_schema_registry,
237 )?;
238
239 stream_source_info.use_schema_registry = use_schema_registry;
240 stream_source_info
241 .row_schema_location
242 .clone_from(&row_schema_location);
243 stream_source_info.proto_message_name = message_name.unwrap_or_default();
244 stream_source_info.key_message_name =
245 get_key_message_name(&mut format_encode_options_to_consume);
246 stream_source_info.name_strategy =
247 name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
248 }
249
250 Some(
251 extract_avro_table_schema(
252 &stream_source_info,
253 &options_with_secret,
254 &mut format_encode_options_to_consume,
255 matches!(format, Format::Debezium),
256 )
257 .await?,
258 )
259 }
260 (Format::Plain, Encode::Csv) => {
261 let chars =
262 consume_string_from_options(&mut format_encode_options_to_consume, "delimiter")?;
263 let delimiter = get_delimiter(chars.as_str()).context("failed to parse delimiter")?;
264 let has_header = try_consume_string_from_options(
265 &mut format_encode_options_to_consume,
266 "without_header",
267 )
268 .map(|s| s == "false")
269 .unwrap_or(true);
270
271 if is_kafka && has_header {
272 return Err(RwError::from(ProtocolError(
273 "CSV HEADER is not supported when creating table with Kafka connector"
274 .to_owned(),
275 )));
276 }
277
278 stream_source_info.csv_delimiter = delimiter as i32;
279 stream_source_info.csv_has_header = has_header;
280
281 None
282 }
283 (Format::Plain, Encode::Parquet) => None,
285 (
286 Format::Plain | Format::Upsert | Format::Maxwell | Format::Canal | Format::Debezium,
287 Encode::Json,
288 ) => {
289 if matches!(
290 format_encode.format,
291 Format::Plain | Format::Upsert | Format::Debezium
292 ) {
293 if let Some(value) =
298 format_encode_options_to_consume.get(TimestamptzHandling::OPTION_KEY)
299 {
300 TimestamptzHandling::from_options(value)
301 .map_err(|err| InvalidInputSyntax(err.message))?;
302 }
303 try_consume_string_from_options(
304 &mut format_encode_options_to_consume,
305 TimestamptzHandling::OPTION_KEY,
306 );
307 }
308
309 if let Some(config) =
310 PulsarSchemaConfig::from_options(&format_encode_options_to_consume)?
311 {
312 Some(
313 extract_pulsar_json_table_schema(
314 &config,
315 &options_with_secret,
316 &mut format_encode_options_to_consume,
317 )
318 .await?,
319 )
320 } else {
321 let schema_config =
322 get_json_schema_location(&mut format_encode_options_to_consume)?;
323 stream_source_info.use_schema_registry =
324 json_schema_infer_use_schema_registry(&schema_config);
325
326 extract_json_table_schema(
327 &schema_config,
328 &options_with_secret,
329 &mut format_encode_options_to_consume,
330 )
331 .await?
332 }
333 }
334 (Format::None, Encode::None) => {
335 if options_with_secret.is_iceberg_connector() {
336 Some(
337 extract_iceberg_columns(&options_with_secret)
338 .await
339 .map_err(|err| ProtocolError(err.to_report_string()))?,
340 )
341 } else if options_with_secret
342 .get(UPSTREAM_SOURCE_KEY)
343 .is_some_and(|s| s.eq_ignore_ascii_case(ADBC_SNOWFLAKE_CONNECTOR))
344 {
345 Some(
346 extract_adbc_snowflake_columns(&options_with_secret)
347 .await
348 .map_err(|err| ProtocolError(err.to_report_string()))?,
349 )
350 } else {
351 None
352 }
353 }
354 (format, encoding) => {
355 return Err(RwError::from(ProtocolError(format!(
356 "Unknown combination {:?} {:?}",
357 format, encoding
358 ))));
359 }
360 };
361
362 if !format_encode_options_to_consume.is_empty() {
363 let err_string = format!(
364 "Get unknown format_encode_options for {:?} {:?}: {}",
365 format_encode.format,
366 format_encode.row_encode,
367 format_encode_options_to_consume
368 .keys()
369 .cloned()
370 .collect::<Vec<String>>()
371 .join(","),
372 );
373 session.notice_to_user(err_string);
374 }
375 Ok((columns, stream_source_info))
376}
377
378fn bind_columns_from_source_for_cdc(
379 session: &SessionImpl,
380 format_encode: &FormatEncodeOptions,
381) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
382 let with_options = WithOptions::try_from(format_encode.row_options())?;
383 if !with_options.connection_ref().is_empty() {
384 return Err(RwError::from(NotSupported(
385 "CDC connector does not support connection ref yet".to_owned(),
386 "Explicitly specify the connection in WITH clause".to_owned(),
387 )));
388 }
389 let (format_encode_options, format_encode_secret_refs) =
390 resolve_secret_ref_in_with_options(with_options, session)?.into_parts();
391
392 let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
394 format_encode_options.clone(),
395 format_encode_secret_refs.clone(),
396 )?;
397
398 match (&format_encode.format, &format_encode.row_encode) {
399 (Format::Plain, Encode::Json) => (),
400 (format, encoding) => {
401 return Err(RwError::from(ProtocolError(format!(
403 "Row format for CDC connectors should be either omitted or set to `FORMAT PLAIN ENCODE JSON`, got: {:?} {:?}",
404 format, encoding
405 ))));
406 }
407 };
408
409 let columns = ColumnCatalog::debezium_cdc_source_cols().to_vec();
410 let schema_config = get_json_schema_location(&mut format_encode_options_to_consume)?;
411
412 let stream_source_info = StreamSourceInfo {
413 format: format_to_prost(&format_encode.format) as i32,
414 row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
415 format_encode_options,
416 use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
417 cdc_source_job: true,
418 is_distributed: false,
419 format_encode_secret_refs,
420 ..Default::default()
421 };
422 if !format_encode_options_to_consume.is_empty() {
423 let err_string = format!(
424 "Get unknown format_encode_options for {:?} {:?}: {}",
425 format_encode.format,
426 format_encode.row_encode,
427 format_encode_options_to_consume
428 .keys()
429 .cloned()
430 .collect::<Vec<String>>()
431 .join(","),
432 );
433 session.notice_to_user(err_string);
434 }
435 Ok((Some(columns), stream_source_info))
436}
437
438pub(super) fn format_to_prost(format: &Format) -> FormatType {
439 match format {
440 Format::Native => FormatType::Native,
441 Format::Plain => FormatType::Plain,
442 Format::Upsert => FormatType::Upsert,
443 Format::Debezium => FormatType::Debezium,
444 Format::DebeziumMongo => FormatType::DebeziumMongo,
445 Format::Maxwell => FormatType::Maxwell,
446 Format::Canal => FormatType::Canal,
447 Format::None => FormatType::None,
448 }
449}
450pub(super) fn row_encode_to_prost(row_encode: &Encode) -> EncodeType {
451 match row_encode {
452 Encode::Native => EncodeType::Native,
453 Encode::Json => EncodeType::Json,
454 Encode::Avro => EncodeType::Avro,
455 Encode::Protobuf => EncodeType::Protobuf,
456 Encode::Csv => EncodeType::Csv,
457 Encode::Bytes => EncodeType::Bytes,
458 Encode::Template => EncodeType::Template,
459 Encode::Parquet => EncodeType::Parquet,
460 Encode::None => EncodeType::None,
461 Encode::Text => EncodeType::Text,
462 }
463}
464
465pub fn get_schema_location(
466 format_encode_options: &mut BTreeMap<String, String>,
467) -> Result<(String, bool)> {
468 let schema_location = try_consume_string_from_options(format_encode_options, "schema.location");
469 let schema_registry = try_consume_string_from_options(format_encode_options, "schema.registry");
470 match (schema_location, schema_registry) {
471 (None, None) => Err(RwError::from(ProtocolError(
472 "missing either a schema location or a schema registry".to_owned(),
473 ))),
474 (None, Some(schema_registry)) => Ok((schema_registry, true)),
475 (Some(schema_location), None) => Ok((schema_location, false)),
476 (Some(_), Some(_)) => Err(RwError::from(ProtocolError(
477 "only need either the schema location or the schema registry".to_owned(),
478 ))),
479 }
480}
481
482pub fn schema_has_schema_registry(schema: &FormatEncodeOptions) -> bool {
483 match schema.row_encode {
484 Encode::Avro | Encode::Protobuf => true,
485 Encode::Json => {
486 let mut options = WithOptions::try_from(schema.row_options()).unwrap();
487 uses_pulsar_schema(&options, options.secret_ref())
488 || matches!(get_json_schema_location(options.inner_mut()), Ok(Some(_)))
489 }
490 _ => false,
491 }
492}
493
494#[inline]
495fn get_name_strategy_or_default(name_strategy: Option<String>) -> Result<Option<i32>> {
496 match name_strategy {
497 None => Ok(None),
498 Some(name) => Ok(Some(name_strategy_from_str(name.as_str())
499 .ok_or_else(|| RwError::from(ProtocolError(format!("\
500 expect strategy name in topic_name_strategy, record_name_strategy and topic_record_name_strategy, but got {}", name))))? as i32)),
501 }
502}