1use std::collections::{BTreeMap, HashMap, HashSet};
16use std::rc::Rc;
17use std::sync::LazyLock;
18
19use anyhow::Context;
20use either::Either;
21use external_schema::debezium::extract_debezium_avro_table_pk_columns;
22use external_schema::nexmark::check_nexmark_schema;
23use itertools::Itertools;
24use maplit::{convert_args, hashmap, hashset};
25use pgwire::pg_response::{PgResponse, StatementType};
26use risingwave_common::array::arrow::{IcebergArrowConvert, arrow_schema_iceberg};
27use risingwave_common::bail_not_implemented;
28use risingwave_common::catalog::{
29 CdcTableDesc, ColumnCatalog, ColumnDesc, ColumnId, INITIAL_SOURCE_VERSION_ID,
30 KAFKA_TIMESTAMP_COLUMN_NAME, ROW_ID_COLUMN_NAME, TableId, debug_assert_column_ids_distinct,
31};
32use risingwave_common::license::Feature;
33use risingwave_common::secret::LocalSecretManager;
34use risingwave_common::system_param::reader::SystemParamsRead;
35use risingwave_common::types::DataType;
36use risingwave_common::util::iter_util::ZipEqFast;
37use risingwave_connector::parser::additional_columns::{
38 build_additional_column_desc, source_add_partition_offset_cols,
39};
40use risingwave_connector::parser::{
41 AvroParserConfig, DEBEZIUM_IGNORE_KEY, DebeziumAvroParserConfig, ProtobufParserConfig,
42 SchemaLocation, SpecificParserConfig, TimestamptzHandling,
43 fetch_json_schema_and_map_to_columns, fetch_pulsar_json_schema_and_map_to_columns,
44};
45use risingwave_connector::schema::AWS_GLUE_SCHEMA_ARN_KEY;
46use risingwave_connector::schema::pulsar_schema::{
47 PULSAR_SCHEMA_AUTH_TOKEN_KEY, PULSAR_SCHEMA_URL_KEY, PulsarSchemaConfig,
48};
49use risingwave_connector::schema::schema_registry::{
50 SCHEMA_REGISTRY_BACKOFF_DURATION_KEY, SCHEMA_REGISTRY_BACKOFF_FACTOR_KEY,
51 SCHEMA_REGISTRY_CA_PEM_PATH, SCHEMA_REGISTRY_MAX_DELAY_KEY, SCHEMA_REGISTRY_PASSWORD,
52 SCHEMA_REGISTRY_RETRIES_MAX_KEY, SCHEMA_REGISTRY_USERNAME, SchemaRegistryConfig,
53 name_strategy_from_str,
54};
55use risingwave_connector::source::cdc::{
56 CDC_BACKFILL_ENABLE_KEY, CDC_MONGODB_STRONG_SCHEMA_KEY,
57 CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY, CDC_SHARING_MODE_KEY, CDC_SNAPSHOT_BACKFILL,
58 CDC_SNAPSHOT_MODE_KEY, CDC_TRANSACTIONAL_KEY, CDC_WAIT_FOR_STREAMING_START_TIMEOUT,
59 CdcSourceDdlValidator, CdcSourceRuntimeFlags,
60};
61use risingwave_connector::source::datagen::DATAGEN_CONNECTOR;
62use risingwave_connector::source::iceberg::ICEBERG_CONNECTOR;
63use risingwave_connector::source::nexmark::source::{EventType, get_event_data_types_with_names};
64use risingwave_connector::source::test_source::TEST_CONNECTOR;
65pub use risingwave_connector::source::{
66 ADBC_SNOWFLAKE_CONNECTOR, UPSTREAM_SOURCE_KEY, WEBHOOK_CONNECTOR,
67};
68use risingwave_connector::source::{
69 AZBLOB_CONNECTOR, ConnectorProperties, GCS_CONNECTOR, GOOGLE_PUBSUB_CONNECTOR, KAFKA_CONNECTOR,
70 KINESIS_CONNECTOR, LEGACY_S3_CONNECTOR, MQTT_CONNECTOR, NATS_CONNECTOR, NEXMARK_CONNECTOR,
71 OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR, PULSAR_CONNECTOR,
72};
73use risingwave_connector::{AUTO_SCHEMA_CHANGE_KEY, WithPropertiesExt, dispatch_cdc_source_type};
74use risingwave_pb::catalog::connection_params::PbConnectionType;
75use risingwave_pb::catalog::{PbSchemaRegistryNameStrategy, StreamSourceInfo, WatermarkDesc};
76use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
77use risingwave_pb::plan_common::source_ingest_mode::{IngestMode, SourceIngestModeStreaming};
78use risingwave_pb::plan_common::{EncodeType, FormatType, SourceIngestMode};
79use risingwave_pb::stream_plan::PbStreamFragmentGraph;
80use risingwave_pb::telemetry::TelemetryDatabaseObject;
81use risingwave_sqlparser::ast::{
82 ColumnDef, ColumnOption, CreateSourceStatement, Encode, Format, FormatEncodeOptions,
83 ObjectName, SourceWatermark, SqlOptionValue, TableConstraint, Value, get_delimiter,
84};
85use risingwave_sqlparser::parser::{IncludeOption, IncludeOptionItem};
86use thiserror_ext::AsReport;
87
88use super::RwPgResponse;
89use crate::binder::Binder;
90use crate::catalog::CatalogError;
91use crate::catalog::root_catalog::SchemaPath;
92use crate::catalog::source_catalog::SourceCatalog;
93use crate::error::ErrorCode::{self, Deprecated, InvalidInputSyntax, NotSupported, ProtocolError};
94use crate::error::{Result, RwError};
95use crate::expr::{Expr, ExprRewriter, SessionTimezone};
96use crate::handler::HandlerArgs;
97use crate::handler::cdc::{
98 bind_cdc_pk_comparisons_externally, bind_cdc_table_schema, bind_cdc_table_schema_externally,
99 derive_connect_properties_for_cdc_table, not_null_check_for_cdc_table,
100 reject_pk_filtered_by_debezium_column_filter, sanity_check_for_table_on_cdc_source,
101};
102use crate::handler::create_table::{
103 ColumnIdGenerator, bind_pk_and_row_id_on_relation, bind_sql_column_constraints,
104 bind_sql_columns, bind_sql_pk_names, bind_table_constraints, check_cdc_source_select_privilege,
105};
106use crate::handler::util::{
107 SourceSchemaCompatExt, check_connector_match_connection_type, ensure_connection_type_allowed,
108 ensure_local_fs_connector_allowed,
109};
110use crate::optimizer::plan_node::generic::SourceNodeKind;
111use crate::optimizer::plan_node::{BackfillType, LogicalSource, ToStream, ToStreamContext};
112use crate::session::SessionImpl;
113use crate::session::current::notice_to_user;
114use crate::utils::{
115 OverwriteOptions, resolve_connection_ref_and_secret_ref, resolve_privatelink_in_with_option,
116 resolve_secret_ref_in_with_options, resolve_source_ingest_mode_in_with_option,
117};
118use crate::{OptimizerContext, WithOptions, WithOptionsSecResolved, bind_data_type, build_graph};
119
120mod external_schema;
121pub use external_schema::{
122 bind_columns_from_source, get_schema_location, schema_has_schema_registry,
123};
124use external_schema::{format_to_prost, row_encode_to_prost};
125mod validate;
126pub use validate::validate_compatibility;
127use validate::{SOURCE_ALLOWED_CONNECTION_CONNECTOR, SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY};
128mod additional_column;
129use additional_column::check_and_add_timestamp_column;
130pub use additional_column::handle_addition_columns;
131use risingwave_common::catalog::ICEBERG_SOURCE_PREFIX;
132use risingwave_common::id::SourceId;
133
134use crate::stream_fragmenter::GraphJobType;
135
136fn non_generated_sql_columns(columns: &[ColumnDef]) -> Vec<ColumnDef> {
137 columns
138 .iter()
139 .filter(|c| !c.is_generated())
140 .cloned()
141 .collect()
142}
143
144fn try_consume_string_from_options(
145 format_encode_options: &mut BTreeMap<String, String>,
146 key: &str,
147) -> Option<String> {
148 format_encode_options.remove(key)
149}
150
151fn try_consume_schema_registry_config_from_options(
152 format_encode_options: &mut BTreeMap<String, String>,
153) {
154 [
155 SCHEMA_REGISTRY_USERNAME,
156 SCHEMA_REGISTRY_PASSWORD,
157 SCHEMA_REGISTRY_CA_PEM_PATH,
158 SCHEMA_REGISTRY_MAX_DELAY_KEY,
159 SCHEMA_REGISTRY_BACKOFF_DURATION_KEY,
160 SCHEMA_REGISTRY_BACKOFF_FACTOR_KEY,
161 SCHEMA_REGISTRY_RETRIES_MAX_KEY,
162 ]
163 .iter()
164 .for_each(|key| {
165 try_consume_string_from_options(format_encode_options, key);
166 });
167}
168
169pub(crate) fn uses_pulsar_schema<V>(
171 format_encode_options: &BTreeMap<String, String>,
172 format_encode_secret_refs: &BTreeMap<String, V>,
173) -> bool {
174 format_encode_options.contains_key(PULSAR_SCHEMA_URL_KEY)
175 || format_encode_secret_refs.contains_key(PULSAR_SCHEMA_URL_KEY)
176}
177
178fn try_consume_pulsar_schema_config_from_options(
179 format_encode_options: &mut BTreeMap<String, String>,
180) {
181 for key in [PULSAR_SCHEMA_URL_KEY, PULSAR_SCHEMA_AUTH_TOKEN_KEY] {
182 try_consume_string_from_options(format_encode_options, key);
183 }
184}
185
186fn consume_string_from_options(
187 format_encode_options: &mut BTreeMap<String, String>,
188 key: &str,
189) -> Result<String> {
190 try_consume_string_from_options(format_encode_options, key).ok_or(RwError::from(ProtocolError(
191 format!("missing field {} in options", key),
192 )))
193}
194
195fn consume_aws_config_from_options(format_encode_options: &mut BTreeMap<String, String>) {
196 format_encode_options.retain(|key, _| !key.starts_with("aws."))
197}
198
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
200pub enum CreateSourceType {
201 SharedCdc,
202 SharedNonCdc,
204 NonShared,
205 Table,
207}
208
209impl CreateSourceType {
210 pub fn for_newly_created(
212 session: &SessionImpl,
213 with_properties: &impl WithPropertiesExt,
214 ) -> Self {
215 if with_properties.is_shareable_cdc_connector() {
216 CreateSourceType::SharedCdc
217 } else if with_properties.is_shareable_non_cdc_connector()
218 && session
219 .env()
220 .streaming_config()
221 .developer
222 .enable_shared_source
223 && session.config().streaming_use_shared_source()
224 {
225 CreateSourceType::SharedNonCdc
226 } else {
227 CreateSourceType::NonShared
228 }
229 }
230
231 pub fn for_replace(catalog: &SourceCatalog) -> Self {
232 if !catalog.info.is_shared() {
233 CreateSourceType::NonShared
234 } else if catalog.with_properties.is_shareable_cdc_connector() {
235 CreateSourceType::SharedCdc
236 } else {
237 CreateSourceType::SharedNonCdc
238 }
239 }
240
241 pub fn is_shared(&self) -> bool {
242 matches!(
243 self,
244 CreateSourceType::SharedCdc | CreateSourceType::SharedNonCdc
245 )
246 }
247}
248
249pub(crate) fn bind_all_columns(
251 format_encode: &FormatEncodeOptions,
252 cols_from_source: Option<Vec<ColumnCatalog>>,
253 cols_from_sql: Vec<ColumnCatalog>,
254 col_defs_from_sql: &[ColumnDef],
255 wildcard_idx: Option<usize>,
256 sql_column_strategy: SqlColumnStrategy,
257) -> Result<Vec<ColumnCatalog>> {
258 if let Some(cols_from_source) = cols_from_source {
259 let generated_cols_from_sql = cols_from_sql
262 .iter()
263 .filter(|c| {
264 col_defs_from_sql
265 .iter()
266 .find(|d| d.name.real_value() == c.name())
267 .unwrap()
268 .is_generated()
269 })
270 .cloned()
271 .collect_vec();
272
273 match sql_column_strategy {
274 SqlColumnStrategy::FollowUnchecked => {
276 assert!(
277 wildcard_idx.is_none(),
278 "wildcard still exists while strategy is Follows, not correctly purified?"
279 );
280 return Ok(cols_from_sql);
281 }
282
283 SqlColumnStrategy::Ignore => {}
285
286 SqlColumnStrategy::FollowChecked => {
287 let has_regular_cols_from_sql =
288 generated_cols_from_sql.len() != cols_from_sql.len();
289
290 if has_regular_cols_from_sql {
291 if wildcard_idx.is_some() {
292 return Err(RwError::from(NotSupported(
294 "When there's a wildcard (\"*\"), \
295 only generated columns are allowed in user-defined schema from SQL"
296 .to_owned(),
297 "Remove the non-generated columns".to_owned(),
298 )));
299 } else {
300 for col in &cols_from_sql {
303 if generated_cols_from_sql.contains(col) {
304 continue;
305 }
306 let Some(col_from_source) =
307 cols_from_source.iter().find(|c| c.name() == col.name())
308 else {
309 return Err(RwError::from(ProtocolError(format!(
310 "Column \"{}\" is defined in SQL but not found in the source",
311 col.name()
312 ))));
313 };
314
315 if col_from_source.data_type() != col.data_type() {
316 return Err(RwError::from(ProtocolError(format!(
317 "Data type mismatch for column \"{}\". \
318 Defined in SQL as \"{}\", but found in the source as \"{}\"",
319 col.name(),
320 col.data_type(),
321 col_from_source.data_type()
322 ))));
323 }
324 }
325 return Ok(cols_from_sql);
326 }
327 } else {
328 if wildcard_idx.is_some() {
329 } else {
333 notice_to_user("\
341 Neither wildcard (\"*\") nor regular (non-generated) columns appear in the user-defined schema from SQL. \
342 For backward compatibility, all columns from the source will be included at the beginning. \
343 For clarity, consider adding a wildcard (\"*\") to indicate where the columns from the source should be included, \
344 or specifying the columns you want to include from the source.
345 ");
346 }
347 }
348 }
349 }
350
351 let wildcard_idx = wildcard_idx.unwrap_or(0).min(generated_cols_from_sql.len());
356
357 let mut merged_cols = generated_cols_from_sql;
359 let merged_cols_r = merged_cols.split_off(wildcard_idx);
360 merged_cols.extend(cols_from_source);
361 merged_cols.extend(merged_cols_r);
362
363 Ok(merged_cols)
364 } else {
365 if wildcard_idx.is_some() {
366 return Err(RwError::from(NotSupported(
367 "Wildcard in user-defined schema is only allowed when there exists columns from external schema".to_owned(),
368 "Remove the wildcard or use a source with external schema".to_owned(),
369 )));
370 }
371 let non_generated_sql_defined_columns = non_generated_sql_columns(col_defs_from_sql);
372
373 match (&format_encode.format, &format_encode.row_encode) {
374 (Format::DebeziumMongo, Encode::Json) => {
375 let strong_schema = format_encode
376 .row_options
377 .iter()
378 .find(|k| k.name.real_value().to_lowercase() == CDC_MONGODB_STRONG_SCHEMA_KEY)
379 .map(|k| matches!(k.value, SqlOptionValue::Value(Value::Boolean(true))))
380 .unwrap_or(false);
381
382 if strong_schema {
384 let (_, id_column) = non_generated_sql_defined_columns
385 .iter()
386 .enumerate()
387 .find(|(idx, col)| *idx == 0 && col.name.real_value() == "_id")
388 .ok_or_else(|| {
389 RwError::from(ProtocolError(
390 "The `_id` column of the source with row format DebeziumMongoJson must be defined as the first column in SQL".to_owned(),
391 ))
392 })?;
393
394 let id_data_type = bind_data_type(id_column.data_type.as_ref().unwrap())?;
395 if !matches!(
396 id_data_type,
397 DataType::Varchar | DataType::Int32 | DataType::Int64 | DataType::Jsonb
398 ) {
399 return Err(RwError::from(ProtocolError(
400 "the `_id` column of the source with row format DebeziumMongoJson must be [Jsonb | Varchar | Int32 | Int64]".to_owned(),
401 )));
402 }
403
404 let mut columns = Vec::with_capacity(non_generated_sql_defined_columns.len());
405 columns.push(
406 ColumnCatalog {
408 column_desc: ColumnDesc::named("_id", 0.into(), id_data_type),
409 is_hidden: false,
410 },
411 );
412
413 for (idx, col) in non_generated_sql_defined_columns
415 .into_iter()
416 .skip(1)
418 .enumerate()
419 {
420 columns.push(ColumnCatalog {
421 column_desc: ColumnDesc::named(
422 col.name.real_value(),
423 (idx as i32).into(),
424 bind_data_type(col.data_type.as_ref().unwrap())?,
425 ),
426 is_hidden: false,
427 });
428 }
429
430 return Ok(columns);
431 }
432
433 let mut columns = vec![
434 ColumnCatalog {
435 column_desc: ColumnDesc::named("_id", 0.into(), DataType::Varchar),
436 is_hidden: false,
437 },
438 ColumnCatalog {
439 column_desc: ColumnDesc::named("payload", 0.into(), DataType::Jsonb),
440 is_hidden: false,
441 },
442 ];
443
444 if non_generated_sql_defined_columns.len() != 2
445 || non_generated_sql_defined_columns[0].name.real_value() != columns[0].name()
446 || non_generated_sql_defined_columns[1].name.real_value() != columns[1].name()
447 {
448 return Err(RwError::from(ProtocolError(
449 "the not generated columns of the source with row format DebeziumMongoJson
450 must be (_id [Jsonb | Varchar | Int32 | Int64], payload jsonb)."
451 .to_owned(),
452 )));
453 }
454 let key_data_type = bind_data_type(
456 non_generated_sql_defined_columns[0]
457 .data_type
458 .as_ref()
459 .unwrap(),
460 )?;
461 match key_data_type {
462 DataType::Jsonb | DataType::Varchar | DataType::Int32 | DataType::Int64 => {
463 columns[0].column_desc.data_type = key_data_type;
464 }
465 _ => {
466 return Err(RwError::from(ProtocolError(
467 "the `_id` column of the source with row format DebeziumMongoJson
468 must be [Jsonb | Varchar | Int32 | Int64]"
469 .to_owned(),
470 )));
471 }
472 }
473
474 let value_data_type = bind_data_type(
476 non_generated_sql_defined_columns[1]
477 .data_type
478 .as_ref()
479 .unwrap(),
480 )?;
481 if !matches!(value_data_type, DataType::Jsonb) {
482 return Err(RwError::from(ProtocolError(
483 "the `payload` column of the source with row format DebeziumMongoJson
484 must be Jsonb datatype"
485 .to_owned(),
486 )));
487 }
488 Ok(columns)
489 }
490 (Format::Plain, Encode::Bytes) => {
491 let err = Err(RwError::from(ProtocolError(
492 "ENCODE BYTES only accepts one BYTEA type column".to_owned(),
493 )));
494 if non_generated_sql_defined_columns.len() == 1 {
495 let col_data_type = bind_data_type(
497 non_generated_sql_defined_columns[0]
498 .data_type
499 .as_ref()
500 .unwrap(),
501 )?;
502 if col_data_type == DataType::Bytea {
503 Ok(cols_from_sql)
504 } else {
505 err
506 }
507 } else {
508 err
509 }
510 }
511 (_, _) => Ok(cols_from_sql),
512 }
513 }
514}
515
516fn hint_format_encode(format_encode: &FormatEncodeOptions) -> String {
518 format!(
519 r#"Hint: For FORMAT {0} ENCODE {1}, INCLUDE KEY must be specified and the key column must be used as primary key.
520example:
521 CREATE TABLE <table_name> ( PRIMARY KEY ([rw_key | <key_name>]) )
522 INCLUDE KEY [AS <key_name>]
523 WITH (...)
524 FORMAT {0} ENCODE {1}{2}
525"#,
526 format_encode.format,
527 format_encode.row_encode,
528 if format_encode.row_encode == Encode::Json || format_encode.row_encode == Encode::Bytes {
529 "".to_owned()
530 } else {
531 " (...)".to_owned()
532 }
533 )
534}
535
536pub(crate) async fn bind_source_pk(
539 format_encode: &FormatEncodeOptions,
540 source_info: &StreamSourceInfo,
541 columns: &mut [ColumnCatalog],
542 sql_defined_pk_names: Vec<String>,
543 with_properties: &WithOptionsSecResolved,
544) -> Result<Vec<String>> {
545 if with_properties.is_iceberg_update_source() {
546 if sql_defined_pk_names.is_empty() {
547 return Err(InvalidInputSyntax(
548 "Iceberg streaming_updates requires a SQL PRIMARY KEY for retracting ingestion"
549 .to_owned(),
550 )
551 .into());
552 }
553 return Ok(sql_defined_pk_names);
554 }
555 let sql_defined_pk = !sql_defined_pk_names.is_empty();
556 let include_key_column_name: Option<String> = {
557 columns.iter().find_map(|catalog| {
560 if matches!(
561 catalog.column_desc.additional_column.column_type,
562 Some(AdditionalColumnType::Key(_))
563 ) {
564 Some(catalog.name().to_owned())
565 } else {
566 None
567 }
568 })
569 };
570 let additional_column_names = columns
571 .iter()
572 .filter_map(|col| {
573 if col.column_desc.additional_column.column_type.is_some() {
574 Some(col.name().to_owned())
575 } else {
576 None
577 }
578 })
579 .collect_vec();
580
581 let res = match (&format_encode.format, &format_encode.row_encode) {
582 (Format::Native, Encode::Native) | (Format::None, Encode::None) | (Format::Plain, _) => {
583 sql_defined_pk_names
584 }
585
586 (Format::Upsert, Encode::Json | Encode::Avro | Encode::Protobuf) => {
589 if let Some(ref key_column_name) = include_key_column_name
590 && sql_defined_pk
591 {
592 if sql_defined_pk_names.len() != 1
597 || !key_column_name.eq(sql_defined_pk_names[0].as_str())
598 {
599 return Err(RwError::from(ProtocolError(format!(
600 "Only \"{}\" can be used as primary key\n\n{}",
601 key_column_name,
602 hint_format_encode(format_encode)
603 ))));
604 }
605 sql_defined_pk_names
606 } else {
607 return if let Some(include_key_column_name) = include_key_column_name {
609 Err(RwError::from(ProtocolError(format!(
610 "Primary key must be specified to {}\n\n{}",
611 include_key_column_name,
612 hint_format_encode(format_encode)
613 ))))
614 } else {
615 Err(RwError::from(ProtocolError(format!(
616 "INCLUDE KEY clause not set\n\n{}",
617 hint_format_encode(format_encode)
618 ))))
619 };
620 }
621 }
622
623 (Format::Debezium, Encode::Json) => {
624 if !additional_column_names.is_empty() {
625 return Err(RwError::from(ProtocolError(format!(
626 "FORMAT DEBEZIUM forbids additional columns, but got {:?}",
627 additional_column_names
628 ))));
629 }
630 if !sql_defined_pk {
631 return Err(RwError::from(ProtocolError(
632 "Primary key must be specified when creating source with FORMAT DEBEZIUM."
633 .to_owned(),
634 )));
635 }
636 sql_defined_pk_names
637 }
638 (Format::Debezium, Encode::Avro) => {
639 if !additional_column_names.is_empty() {
640 return Err(RwError::from(ProtocolError(format!(
641 "FORMAT DEBEZIUM forbids additional columns, but got {:?}",
642 additional_column_names
643 ))));
644 }
645 if sql_defined_pk {
646 sql_defined_pk_names
647 } else {
648 let pk_names =
649 extract_debezium_avro_table_pk_columns(source_info, with_properties).await?;
650 for pk_name in &pk_names {
652 columns
653 .iter()
654 .find(|c: &&ColumnCatalog| c.name().eq(pk_name))
655 .ok_or_else(|| {
656 RwError::from(ProtocolError(format!(
657 "avro's key column {} not exists in avro's row schema",
658 pk_name
659 )))
660 })?;
661 }
662 pk_names
663 }
664 }
665 (Format::DebeziumMongo, Encode::Json) => {
666 if sql_defined_pk {
667 sql_defined_pk_names
668 } else {
669 vec!["_id".to_owned()]
670 }
671 }
672
673 (Format::Maxwell, Encode::Json) => {
674 if !additional_column_names.is_empty() {
675 return Err(RwError::from(ProtocolError(format!(
676 "FORMAT MAXWELL forbids additional columns, but got {:?}",
677 additional_column_names
678 ))));
679 }
680 if !sql_defined_pk {
681 return Err(RwError::from(ProtocolError(
682 "Primary key must be specified when creating source with FORMAT MAXWELL ENCODE JSON.".to_owned(),
683 )));
684 }
685 sql_defined_pk_names
686 }
687
688 (Format::Canal, Encode::Json) => {
689 if !additional_column_names.is_empty() {
690 return Err(RwError::from(ProtocolError(format!(
691 "FORMAT CANAL forbids additional columns, but got {:?}",
692 additional_column_names
693 ))));
694 }
695 if !sql_defined_pk {
696 return Err(RwError::from(ProtocolError(
697 "Primary key must be specified when creating source with FORMAT CANAL ENCODE JSON.".to_owned(),
698 )));
699 }
700 sql_defined_pk_names
701 }
702 (format, encoding) => {
703 return Err(RwError::from(ProtocolError(format!(
704 "Unknown combination {:?} {:?}",
705 format, encoding
706 ))));
707 }
708 };
709 Ok(res)
710}
711
712pub(super) fn bind_source_watermark(
713 session: &SessionImpl,
714 name: String,
715 source_watermarks: Vec<SourceWatermark>,
716 column_catalogs: &[ColumnCatalog],
717) -> Result<Vec<WatermarkDesc>> {
718 let mut binder = Binder::new_for_ddl(session);
719 binder.bind_columns_to_context(name.clone(), column_catalogs)?;
720
721 let mut session_tz = SessionTimezone::new(session.config().timezone());
722
723 let watermark_descs = source_watermarks
724 .into_iter()
725 .map(|source_watermark| {
726 let col_name = source_watermark.column.real_value();
727 let watermark_idx = binder.get_column_binding_index(name.clone(), &col_name)?;
728
729 let expr = binder.bind_expr(&source_watermark.expr)?;
730 let expr = session_tz.rewrite_expr(expr);
734 let watermark_col_type = column_catalogs[watermark_idx].data_type();
735 let watermark_expr_type = &expr.return_type();
736 if watermark_col_type != watermark_expr_type {
737 Err(RwError::from(ErrorCode::BindError(
738 format!("The return value type of the watermark expression must be identical to the watermark column data type. Current data type of watermark return value: `{}`, column `{}`",watermark_expr_type, watermark_col_type),
739 )))
740 } else {
741 let expr_proto = expr.to_expr_proto();
742 Ok::<_, RwError>(WatermarkDesc {
743 watermark_idx: watermark_idx as u32,
744 expr: Some(expr_proto),
745 with_ttl: source_watermark.with_ttl,
746 })
747 }
748 })
749 .try_collect()?;
750 Ok(watermark_descs)
751}
752
753pub(super) fn check_format_encode(
759 props: &WithOptionsSecResolved,
760 row_id_index: Option<usize>,
761 columns: &[ColumnCatalog],
762) -> Result<()> {
763 let Some(connector) = props.get_connector() else {
764 return Ok(());
765 };
766
767 if connector == NEXMARK_CONNECTOR {
768 check_nexmark_schema(props, row_id_index, columns)
769 } else {
770 Ok(())
771 }
772}
773
774pub fn bind_connector_props(
775 handler_args: &HandlerArgs,
776 format_encode: &FormatEncodeOptions,
777 is_create_source: bool,
778) -> Result<(WithOptions, SourceIngestMode)> {
779 let mut with_properties = handler_args.with_options.clone().into_connector_props();
780 if with_properties.is_iceberg_connector()
783 && with_properties
784 .secret_ref()
785 .contains_key(risingwave_connector::source::iceberg::STREAMING_UPDATES_KEY)
786 {
787 return Err(RwError::from(ErrorCode::InvalidParameterValue(
788 "streaming_updates cannot be provided as a SECRET; use a plaintext 'true' or 'false'"
789 .to_owned(),
790 )));
791 }
792 let (_, secret_refs) =
793 resolve_secret_ref_in_with_options(with_properties.clone(), &handler_args.session)?
794 .into_parts();
795 validate_compatibility(format_encode, &mut with_properties, &secret_refs)?;
796 let ingest_mode = {
797 let ingest_mode = resolve_source_ingest_mode_in_with_option(&mut with_properties)?;
798 if is_create_source && ingest_mode.is_some() {
799 return Err(RwError::from(ProtocolError(
800 "`refresh_mode` only supported for CREATE TABLE".to_owned(),
801 )));
802 }
803
804 ingest_mode.unwrap_or(SourceIngestMode {
805 ingest_mode: Some(IngestMode::Streaming(SourceIngestModeStreaming {})),
806 })
807 };
808
809 let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
810
811 if !is_create_source && with_properties.is_shareable_cdc_connector() {
812 return Err(RwError::from(ProtocolError(format!(
813 "directly creating a CDC table for connector {} is no longer supported; \
814 please `CREATE SOURCE` to create a shared CDC source first, \
815 then `CREATE TABLE ... FROM <source> TABLE '<database>.<table>'`",
816 with_properties.get_connector().unwrap(),
817 ))));
818 }
819 if with_properties.is_cdc_connector() {
820 if with_properties.contains_key(CDC_TRANSACTIONAL_KEY)
821 || with_properties
822 .secret_ref()
823 .contains_key(CDC_TRANSACTIONAL_KEY)
824 {
825 return Err(RwError::from(ErrorCode::InvalidInputSyntax(format!(
826 "'{}' cannot be set: RisingWave enables transaction metadata itself for the connectors that support it",
827 CDC_TRANSACTIONAL_KEY
828 ))));
829 }
830 if with_properties.contains_key(CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY)
831 || with_properties
832 .secret_ref()
833 .contains_key(CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY)
834 {
835 return Err(RwError::from(ErrorCode::InvalidInputSyntax(format!(
836 "'{}' cannot be set: RisingWave sets it itself for the connectors that commit offsets to the upstream database",
837 CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY
838 ))));
839 }
840 if with_properties
841 .secret_ref()
842 .contains_key(AUTO_SCHEMA_CHANGE_KEY)
843 {
844 return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
845 "'{}' cannot be provided as a SECRET; use a plaintext 'true' or 'false'",
846 AUTO_SCHEMA_CHANGE_KEY
847 ))));
848 }
849 if let Some(value) = with_properties.get(AUTO_SCHEMA_CHANGE_KEY)
850 && value.parse::<bool>().map_err(|_| {
851 ErrorCode::InvalidInputSyntax(format!(
852 "invalid value of '{}' option",
853 AUTO_SCHEMA_CHANGE_KEY
854 ))
855 })?
856 {
857 if !dispatch_cdc_source_type!(
858 with_properties.cdc_source_type(),
859 T,
860 T::CAN_AUTO_SCHEMA_CHANGE,
861 false
862 ) {
863 return Err(ErrorCode::NotSupported(
864 format!(
865 "auto schema change for connector {}",
866 with_properties.get_connector().unwrap()
867 ),
868 format!(
869 "Remove the '{}' option or set it to false",
870 AUTO_SCHEMA_CHANGE_KEY
871 ),
872 )
873 .into());
874 }
875 Feature::CdcAutoSchemaChange.check_available()?;
876 }
877 }
878 if is_create_source && create_cdc_source_job {
879 with_properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), CDC_SNAPSHOT_BACKFILL.into());
881 with_properties.insert(CDC_SHARING_MODE_KEY.into(), "true".into());
883 if with_properties.enable_transaction_metadata() {
885 with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
886 }
887 if !with_properties.contains_key(CDC_WAIT_FOR_STREAMING_START_TIMEOUT) {
889 with_properties.insert(
890 CDC_WAIT_FOR_STREAMING_START_TIMEOUT.into(),
891 handler_args
892 .session
893 .config()
894 .cdc_source_wait_streaming_start_timeout()
895 .to_string(),
896 );
897 }
898 }
899 Ok((with_properties, ingest_mode))
900}
901
902pub enum SqlColumnStrategy {
905 FollowUnchecked,
910
911 FollowChecked,
918
919 Ignore,
924}
925
926pub(crate) fn reject_variant_columns(columns: &[ColumnCatalog], context: &str) -> Result<()> {
929 if let Some(col) = columns.iter().find(|c| c.data_type().contains_variant()) {
930 return Err(RwError::from(NotSupported(
931 format!(
932 "VARIANT column \"{}\" is not supported {context} yet",
933 col.name()
934 ),
935 format!("VARIANT columns are not supported {context} yet"),
936 )));
937 }
938 Ok(())
939}
940
941fn reject_variant_columns_for_unsupported_encoding(
947 format_encode: &FormatEncodeOptions,
948 columns_from_sql: &[ColumnCatalog],
949) -> Result<()> {
950 if matches!(
951 (&format_encode.format, &format_encode.row_encode),
952 (Format::None, Encode::None) | (_, Encode::Parquet)
953 ) {
954 return Ok(());
955 }
956 reject_variant_columns(columns_from_sql, "for this source encoding")
957}
958
959#[expect(clippy::too_many_arguments)]
962pub async fn bind_create_source_or_table_with_connector(
963 handler_args: HandlerArgs,
964 full_name: ObjectName,
965 format_encode: FormatEncodeOptions,
966 with_properties: WithOptions,
967 sql_columns_defs: &[ColumnDef],
968 constraints: Vec<TableConstraint>,
969 wildcard_idx: Option<usize>,
970 source_watermarks: Vec<SourceWatermark>,
971 columns_from_resolve_source: Option<Vec<ColumnCatalog>>,
972 source_info: StreamSourceInfo,
973 include_column_options: IncludeOption,
974 col_id_gen: &mut ColumnIdGenerator,
975 create_source_type: CreateSourceType,
976 source_rate_limit: Option<u32>,
977 sql_column_strategy: SqlColumnStrategy,
978 ingest_mode: SourceIngestMode,
979) -> Result<SourceCatalog> {
980 let session = &handler_args.session;
981 let db_name: &str = &session.database();
982 let (schema_name, source_name) = Binder::resolve_schema_qualified_name(db_name, &full_name)?;
983 let (database_id, schema_id) =
984 session.get_database_and_schema_id_for_create(schema_name.clone())?;
985
986 let is_create_source = create_source_type != CreateSourceType::Table;
987
988 if is_create_source {
989 if with_properties.is_batch_connector() {
991 return Err(ErrorCode::BindError(
992 "can't CREATE SOURCE with refreshable batch connector\n\nHint: use CREATE TABLE instead"
993 .to_owned(),
994 )
995 .into());
996 }
997
998 match format_encode.format {
999 Format::Debezium | Format::DebeziumMongo | Format::Maxwell | Format::Canal => {
1001 return Err(ErrorCode::BindError(format!(
1002 "can't CREATE SOURCE with FORMAT {}.\n\nHint: use CREATE TABLE instead\n\n{}",
1003 format_encode.format,
1004 hint_format_encode(&format_encode)
1005 ))
1006 .into());
1007 }
1008 Format::Upsert => {
1010 notice_to_user(format!(
1011 "Streaming queries on sources with `FORMAT {}` may have limitations. If your query isn't supported, consider using `CREATE TABLE` instead.",
1012 format_encode.format
1013 ));
1014 }
1015 _ => {}
1016 }
1017 }
1018
1019 let sql_pk_names = bind_sql_pk_names(sql_columns_defs, bind_table_constraints(&constraints)?)?;
1020
1021 if with_properties.is_iceberg_connector() {
1022 if is_create_source
1023 && !with_properties.is_iceberg_update_source()
1024 && !sql_pk_names.is_empty()
1025 {
1026 return Err(ErrorCode::NotSupported(
1027 "PRIMARY KEY is not supported for Iceberg CREATE SOURCE in continuous ingestion mode."
1028 .to_owned(),
1029 "Iceberg streaming ingestion only supports append-only sources. Remove the PRIMARY KEY clause."
1030 .to_owned(),
1031 )
1032 .into());
1033 }
1034
1035 if !sql_columns_defs.is_empty() {
1039 return Err(RwError::from(InvalidInputSyntax(
1040 r#"Schema is automatically inferred for iceberg source and should not be specified
1041
1042HINT: use `CREATE SOURCE <name> WITH (...)` instead of `CREATE SOURCE <name> (<columns>) WITH (...)`."#.to_owned(),
1043 )));
1044 }
1045 }
1046
1047 if with_properties.is_batch_connector()
1049 && with_properties
1050 .get(UPSTREAM_SOURCE_KEY)
1051 .is_some_and(|s| s.eq_ignore_ascii_case(ADBC_SNOWFLAKE_CONNECTOR))
1052 && !sql_columns_defs.is_empty()
1053 {
1054 return Err(RwError::from(InvalidInputSyntax(
1055 r#"Schema is automatically inferred for ADBC Snowflake source and should not be specified
1056
1057HINT: use `CREATE TABLE <name> WITH (...)` instead of `CREATE TABLE <name> (<columns>) WITH (...)`."#.to_owned(),
1058 )));
1059 }
1060 let columns_from_sql = bind_sql_columns(sql_columns_defs, false)?;
1061
1062 reject_variant_columns_for_unsupported_encoding(&format_encode, &columns_from_sql)?;
1063
1064 let mut columns = bind_all_columns(
1065 &format_encode,
1066 columns_from_resolve_source,
1067 columns_from_sql,
1068 sql_columns_defs,
1069 wildcard_idx,
1070 sql_column_strategy,
1071 )?;
1072
1073 handle_addition_columns(
1075 Some(&format_encode),
1076 &with_properties,
1077 include_column_options,
1078 &mut columns,
1079 false,
1080 )?;
1081
1082 if columns.is_empty() {
1083 return Err(RwError::from(ProtocolError(
1084 "Schema definition is required, either from SQL or schema registry.".to_owned(),
1085 )));
1086 }
1087
1088 if is_create_source {
1090 check_and_add_timestamp_column(&with_properties, &mut columns);
1092
1093 if create_source_type == CreateSourceType::SharedNonCdc {
1096 let (columns_exist, additional_columns) = source_add_partition_offset_cols(
1097 &columns,
1098 &with_properties.get_connector().unwrap(),
1099 true, );
1101 for (existed, c) in columns_exist.into_iter().zip_eq_fast(additional_columns) {
1102 if !existed {
1103 columns.push(ColumnCatalog::hidden(c));
1104 }
1105 }
1106 }
1107 }
1108
1109 let mut with_properties = with_properties;
1111 resolve_privatelink_in_with_option(&mut with_properties)?;
1112
1113 if session
1115 .env()
1116 .system_params_manager()
1117 .get_params()
1118 .load()
1119 .enforce_secret()
1120 && Feature::SecretManagement.check_available().is_ok()
1121 {
1122 ConnectorProperties::enforce_secret_source(&with_properties)?;
1124 }
1125
1126 let (with_properties, connection_type, connector_conn_ref) =
1127 resolve_connection_ref_and_secret_ref(
1128 with_properties,
1129 session,
1130 Some(TelemetryDatabaseObject::Source),
1131 )?;
1132 ensure_connection_type_allowed(connection_type, &SOURCE_ALLOWED_CONNECTION_CONNECTOR)?;
1133
1134 if !matches!(connection_type, PbConnectionType::Unspecified) {
1136 let Some(connector) = with_properties.get_connector() else {
1137 return Err(RwError::from(ProtocolError(format!(
1138 "missing field '{}' in WITH clause",
1139 UPSTREAM_SOURCE_KEY
1140 ))));
1141 };
1142 check_connector_match_connection_type(connector.as_str(), &connection_type)?;
1143 }
1144
1145 let pk_names = bind_source_pk(
1146 &format_encode,
1147 &source_info,
1148 &mut columns,
1149 sql_pk_names,
1150 &with_properties,
1151 )
1152 .await?;
1153
1154 if let Some(duplicated_name) = columns.iter().map(|c| c.name()).duplicates().next() {
1157 return Err(ErrorCode::InvalidInputSyntax(format!(
1158 "column \"{}\" specified more than once",
1159 duplicated_name
1160 ))
1161 .into());
1162 }
1163
1164 for c in &mut columns {
1166 let original_data_type = c.data_type().clone();
1167 col_id_gen.generate(c)?;
1168 if is_create_source {
1172 c.column_desc.data_type = original_data_type;
1173 }
1174 }
1175 debug_assert_column_ids_distinct(&columns);
1176
1177 let (mut columns, pk_col_ids, row_id_index) =
1178 bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
1179
1180 let watermark_descs =
1181 bind_source_watermark(session, source_name.clone(), source_watermarks, &columns)?;
1182 assert!(watermark_descs.len() <= 1);
1184 if is_create_source && watermark_descs.iter().any(|d| d.with_ttl) {
1185 return Err(ErrorCode::NotSupported(
1186 "WITH TTL is not supported in WATERMARK clause for CREATE SOURCE.".to_owned(),
1187 "Use `CREATE TABLE ... WATERMARK ... WITH TTL` instead.".to_owned(),
1188 )
1189 .into());
1190 }
1191
1192 let append_only = row_id_index.is_some();
1193 if is_create_source && !append_only && !watermark_descs.is_empty() {
1194 return Err(ErrorCode::NotSupported(
1195 "Defining watermarks on source requires the source connector to be append only."
1196 .to_owned(),
1197 "Use the key words `FORMAT PLAIN`".to_owned(),
1198 )
1199 .into());
1200 }
1201
1202 bind_sql_column_constraints(
1203 session,
1204 source_name.clone(),
1205 &mut columns,
1206 sql_columns_defs,
1208 &pk_col_ids,
1209 )?;
1210 check_format_encode(&with_properties, row_id_index, &columns)?;
1211
1212 let definition = handler_args.normalized_sql.clone();
1213
1214 let associated_table_id = if is_create_source {
1215 None
1216 } else {
1217 Some(TableId::placeholder())
1218 };
1219 let source = SourceCatalog {
1220 id: SourceId::placeholder(),
1221 name: source_name,
1222 schema_id,
1223 database_id,
1224 columns,
1225 pk_col_ids,
1226 append_only,
1227 owner: session.user_id(),
1228 info: source_info,
1229 row_id_index,
1230 with_properties,
1231 watermark_descs,
1232 associated_table_id,
1233 definition,
1234 connection_id: connector_conn_ref,
1235 created_at_epoch: None,
1236 initialized_at_epoch: None,
1237 version: INITIAL_SOURCE_VERSION_ID,
1238 created_at_cluster_version: None,
1239 initialized_at_cluster_version: None,
1240 rate_limit: source_rate_limit,
1241 ingest_mode: Some(ingest_mode),
1242 };
1243 source.validate_iceberg_update_source()?;
1244 Ok(source)
1245}
1246
1247pub async fn handle_create_source(
1248 mut handler_args: HandlerArgs,
1249 stmt: CreateSourceStatement,
1250) -> Result<RwPgResponse> {
1251 let session = handler_args.session.clone();
1252
1253 if let Either::Right(resp) = session.check_relation_name_duplicated(
1254 stmt.source_name.clone(),
1255 StatementType::CREATE_SOURCE,
1256 stmt.if_not_exists,
1257 )? {
1258 return Ok(resp);
1259 }
1260
1261 if stmt
1262 .source_name
1263 .base_name()
1264 .starts_with(ICEBERG_SOURCE_PREFIX)
1265 {
1266 return Err(RwError::from(InvalidInputSyntax(format!(
1267 "Source name cannot start with reserved prefix '{}'",
1268 ICEBERG_SOURCE_PREFIX
1269 ))));
1270 }
1271
1272 if stmt.cdc_table_info.is_some() {
1273 return handle_create_cdc_table_source(handler_args, stmt).await;
1274 }
1275
1276 let overwrite_options = OverwriteOptions::new(&mut handler_args)?;
1277
1278 if handler_args.with_options.is_empty() {
1279 return Err(RwError::from(InvalidInputSyntax(
1280 "missing WITH clause".to_owned(),
1281 )));
1282 }
1283
1284 if overwrite_options.source_rate_limit == Some(0)
1285 && dispatch_cdc_source_type!(
1286 handler_args.with_options.cdc_source_type(),
1287 T,
1288 T::NEEDS_NON_EMPTY_INITIAL_OFFSET,
1289 false
1290 )
1291 {
1292 let connector = handler_args.with_options.get_connector().unwrap();
1293 return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
1294 "`source_rate_limit` cannot be 0 when creating a `{connector}` source because source creation must wait for the initial CDC offset."
1295 ))));
1296 }
1297
1298 let format_encode = stmt.format_encode.into_v2_with_warning();
1299 let (with_properties, ingest_mode) = bind_connector_props(&handler_args, &format_encode, true)?;
1300 if let Some(connector) = with_properties.get_connector() {
1301 ensure_local_fs_connector_allowed(&session, &connector)?;
1302 }
1303
1304 let (schema_name, _) =
1307 Binder::resolve_schema_qualified_name(&session.database(), &stmt.source_name)?;
1308 session.get_database_and_schema_id_for_create(schema_name)?;
1309
1310 let create_source_type = CreateSourceType::for_newly_created(&session, &*with_properties);
1311 let (columns_from_resolve_source, source_info) = bind_columns_from_source(
1312 &session,
1313 &format_encode,
1314 Either::Left(&with_properties),
1315 create_source_type,
1316 )
1317 .await?;
1318 let mut col_id_gen = ColumnIdGenerator::new_initial();
1319
1320 if stmt.columns.iter().any(|col| {
1321 col.options
1322 .iter()
1323 .any(|def| matches!(def.option, ColumnOption::NotNull))
1324 }) {
1325 return Err(RwError::from(InvalidInputSyntax(
1326 "NOT NULL constraint is not supported in source schema".to_owned(),
1327 )));
1328 }
1329
1330 let source_catalog = bind_create_source_or_table_with_connector(
1331 handler_args.clone(),
1332 stmt.source_name,
1333 format_encode,
1334 with_properties,
1335 &stmt.columns,
1336 stmt.constraints,
1337 stmt.wildcard_idx,
1338 stmt.source_watermarks,
1339 columns_from_resolve_source,
1340 source_info,
1341 stmt.include_column_options,
1342 &mut col_id_gen,
1343 create_source_type,
1344 overwrite_options.source_rate_limit,
1345 SqlColumnStrategy::FollowChecked,
1346 ingest_mode,
1347 )
1348 .await?;
1349
1350 if stmt.temporary {
1352 if session.get_temporary_source(&source_catalog.name).is_some() {
1353 return Err(CatalogError::duplicated("source", source_catalog.name.clone()).into());
1354 }
1355 session.create_temporary_source(source_catalog);
1356 return Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE));
1357 }
1358
1359 let source = source_catalog.to_prost();
1360
1361 let catalog_writer = session.catalog_writer()?;
1362
1363 if create_source_type.is_shared() {
1364 let graph = generate_stream_graph_for_source(handler_args, source_catalog)?;
1365 catalog_writer
1366 .create_source(source, Some(graph), stmt.if_not_exists)
1367 .await?;
1368 } else {
1369 catalog_writer
1371 .create_source(source, None, stmt.if_not_exists)
1372 .await?;
1373 }
1374
1375 Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE))
1376}
1377
1378async fn handle_create_cdc_table_source(
1379 handler_args: HandlerArgs,
1380 stmt: CreateSourceStatement,
1381) -> Result<RwPgResponse> {
1382 let session = handler_args.session.clone();
1383 let cdc_table_info = stmt.cdc_table_info.as_ref().expect("checked by caller");
1384
1385 if stmt.temporary {
1386 return Err(ErrorCode::NotSupported(
1387 "temporary CDC table sources are not supported".to_owned(),
1388 "Remove the TEMPORARY clause".to_owned(),
1389 )
1390 .into());
1391 }
1392
1393 if handler_args.with_options.len() != 1
1394 || !handler_args
1395 .with_options
1396 .value_eq_ignore_case(CDC_BACKFILL_ENABLE_KEY, "false")
1397 || !handler_args.with_options.secret_ref().is_empty()
1398 || !handler_args.with_options.connection_ref().is_empty()
1399 {
1400 return Err(ErrorCode::InvalidInputSyntax(
1401 "CDC table sources currently require exactly `WITH (snapshot = 'false')`".to_owned(),
1402 )
1403 .into());
1404 }
1405
1406 if !stmt.source_watermarks.is_empty() {
1407 return Err(ErrorCode::NotSupported(
1408 "watermarks on CDC table sources are not supported".to_owned(),
1409 "Remove the WATERMARK clause".to_owned(),
1410 )
1411 .into());
1412 }
1413 if stmt.columns.iter().any(|column| column.is_generated()) {
1414 return Err(ErrorCode::NotSupported(
1415 "generated columns on CDC table sources are not supported".to_owned(),
1416 "Define generated expressions in the downstream streaming query".to_owned(),
1417 )
1418 .into());
1419 }
1420 for column in &stmt.columns {
1421 for option in &column.options {
1422 if matches!(
1423 option.option,
1424 ColumnOption::DefaultValue(_) | ColumnOption::DefaultValueInternal { .. }
1425 ) {
1426 return Err(ErrorCode::NotSupported(
1427 "default values on CDC table sources are not supported".to_owned(),
1428 "Remove the default value expression".to_owned(),
1429 )
1430 .into());
1431 }
1432 }
1433 }
1434
1435 sanity_check_for_table_on_cdc_source(
1436 false,
1437 &stmt.columns,
1438 &stmt.wildcard_idx,
1439 &stmt.constraints,
1440 &stmt.source_watermarks,
1441 )?;
1442 not_null_check_for_cdc_table(&stmt.wildcard_idx, &stmt.columns)?;
1443
1444 let db_name = session.database();
1445 let user_name = session.user_name();
1446 let search_path = session.config().search_path();
1447 let (schema_name, source_name) =
1448 Binder::resolve_schema_qualified_name(&db_name, &stmt.source_name)?;
1449 let (database_id, schema_id) =
1450 session.get_database_and_schema_id_for_create(schema_name.clone())?;
1451
1452 let (upstream_schema, upstream_source_name) =
1453 Binder::resolve_schema_qualified_name(&db_name, &cdc_table_info.source_name)?;
1454 let upstream_source = {
1455 let catalog_reader = session.env().catalog_reader().read_guard();
1456 let schema_path = SchemaPath::new(upstream_schema.as_deref(), &search_path, &user_name);
1457 let (source, _) = catalog_reader.get_source_by_name(
1458 &db_name,
1459 schema_path,
1460 upstream_source_name.as_str(),
1461 )?;
1462 source.clone()
1463 };
1464 check_cdc_source_select_privilege(&session, &upstream_source)?;
1465 if !upstream_source.info.is_shared()
1466 || !upstream_source.with_properties.is_shareable_cdc_connector()
1467 || upstream_source.info.external_table.is_some()
1468 {
1469 return Err(ErrorCode::InvalidInputSyntax(format!(
1470 "source `{}` is not a shared CDC source",
1471 cdc_table_info.source_name
1472 ))
1473 .into());
1474 }
1475
1476 let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1477 &upstream_source.with_properties,
1478 cdc_table_info.external_table_name.clone(),
1479 )?;
1480 let (mut columns, pk_names, pk_comparisons) = match stmt.wildcard_idx {
1481 Some(_) => {
1482 bind_cdc_table_schema_externally(cdc_with_options.clone(), &HashSet::new()).await?
1483 }
1484 None => {
1485 let (columns, pk_names) =
1486 bind_cdc_table_schema(&stmt.columns, &stmt.constraints, false)?;
1487 let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1488 cdc_with_options.clone(),
1489 &pk_names,
1490 ))
1491 .await?;
1492 (columns, pk_names, pk_comparisons)
1493 }
1494 };
1495 if pk_names.is_empty() {
1496 return Err(ErrorCode::NotSupported(
1497 "CDC table source without a primary key is not supported".to_owned(),
1498 "Define a primary key on the upstream table or in the source schema".to_owned(),
1499 )
1500 .into());
1501 }
1502 reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
1503
1504 handle_addition_columns(
1505 None,
1506 &cdc_with_options,
1507 stmt.include_column_options.clone(),
1508 &mut columns,
1509 true,
1510 )?;
1511 let mut col_id_gen = ColumnIdGenerator::new_initial();
1512 for column in &mut columns {
1513 col_id_gen.generate(column)?;
1514 }
1515 debug_assert_column_ids_distinct(&columns);
1516 let (columns, pk_col_ids, row_id_index) =
1517 bind_pk_and_row_id_on_relation(columns, pk_names, false)?;
1518 debug_assert!(row_id_index.is_none());
1519
1520 let (connect_properties, secret_refs) = cdc_with_options.into_parts();
1521 let id_to_index = columns
1522 .iter()
1523 .enumerate()
1524 .map(|(index, column)| (column.column_id(), index))
1525 .collect::<HashMap<_, _>>();
1526 let stream_key = pk_col_ids.iter().map(|id| id_to_index[id]).collect_vec();
1527 let pk = stream_key
1528 .iter()
1529 .map(|index| {
1530 risingwave_common::util::sort_util::ColumnOrder::new(
1531 *index,
1532 risingwave_common::util::sort_util::OrderType::ascending(),
1533 )
1534 })
1535 .collect();
1536 let cdc_table_desc = CdcTableDesc {
1537 table_id: TableId::placeholder(),
1538 source_id: upstream_source.id,
1539 external_table_name,
1540 pk,
1541 pk_comparisons,
1542 columns: columns
1543 .iter()
1544 .map(|column| column.column_desc.clone())
1545 .collect(),
1546 stream_key,
1547 connect_properties,
1552 secret_refs,
1553 };
1554
1555 let mut source_info = upstream_source.info.clone();
1556 source_info.cdc_source_job = false;
1557 source_info.is_distributed = false;
1558 source_info.external_table = Some(cdc_table_desc.to_protobuf());
1559
1560 let catalog_with_properties = WithOptionsSecResolved::new(
1563 BTreeMap::from([
1564 (
1565 UPSTREAM_SOURCE_KEY.to_owned(),
1566 upstream_source
1567 .with_properties
1568 .get_connector()
1569 .expect("validated as a CDC source"),
1570 ),
1571 (CDC_BACKFILL_ENABLE_KEY.to_owned(), "false".to_owned()),
1572 ]),
1573 BTreeMap::new(),
1574 );
1575
1576 let source_catalog = SourceCatalog {
1577 id: SourceId::placeholder(),
1578 name: source_name,
1579 schema_id,
1580 database_id,
1581 columns,
1582 pk_col_ids,
1583 append_only: false,
1584 owner: session.user_id(),
1585 info: source_info,
1586 row_id_index: None,
1587 with_properties: catalog_with_properties,
1588 watermark_descs: vec![],
1589 associated_table_id: None,
1590 definition: handler_args.normalized_sql,
1591 connection_id: None,
1592 created_at_epoch: None,
1593 initialized_at_epoch: None,
1594 version: INITIAL_SOURCE_VERSION_ID,
1595 created_at_cluster_version: None,
1596 initialized_at_cluster_version: None,
1597 rate_limit: None,
1598 ingest_mode: upstream_source.ingest_mode,
1599 };
1600
1601 session
1602 .catalog_writer()?
1603 .create_source(source_catalog.to_prost(), None, stmt.if_not_exists)
1604 .await?;
1605 Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE))
1606}
1607
1608pub(super) fn generate_stream_graph_for_source(
1609 handler_args: HandlerArgs,
1610 source_catalog: SourceCatalog,
1611) -> Result<PbStreamFragmentGraph> {
1612 let context = OptimizerContext::from_handler_args(handler_args)?;
1613 let source_node = LogicalSource::with_catalog(
1614 Rc::new(source_catalog),
1615 SourceNodeKind::CreateSharedSource,
1616 context.into(),
1617 None,
1618 )?;
1619
1620 let stream_plan = source_node.to_stream(&mut ToStreamContext::new_with_backfill_type(
1621 false,
1622 BackfillType::ArrangementBackfill,
1625 ))?;
1626 let graph = build_graph(stream_plan, Some(GraphJobType::Source))?;
1627 Ok(graph)
1628}
1629
1630#[cfg(all(test, not(madsim)))]
1631#[path = "create_source/iceberg_update_test.rs"]
1632mod iceberg_update_tests;
1633
1634#[cfg(test)]
1635pub mod tests {
1636 use std::collections::HashMap;
1637 use std::sync::Arc;
1638
1639 use risingwave_common::catalog::{
1640 DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, ROW_ID_COLUMN_NAME,
1641 };
1642 use risingwave_common::config::FrontendConfig;
1643 use risingwave_common::types::{DataType, StructType};
1644 use risingwave_connector::parser::additional_columns::get_kafka_header_item_datatype;
1645 use risingwave_pb::plan_common::EncodeType;
1646
1647 use crate::catalog::root_catalog::SchemaPath;
1648 use crate::catalog::source_catalog::SourceCatalog;
1649 use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
1650
1651 const GET_COLUMN_FROM_CATALOG: fn(&Arc<SourceCatalog>) -> HashMap<&str, DataType> =
1652 |catalog: &Arc<SourceCatalog>| -> HashMap<&str, DataType> {
1653 catalog
1654 .columns
1655 .iter()
1656 .map(|col| (col.name(), col.data_type().clone()))
1657 .collect::<HashMap<&str, DataType>>()
1658 };
1659
1660 #[tokio::test]
1661 async fn test_create_source_handler() {
1662 let proto_file = create_proto_file(PROTO_FILE_DATA);
1663 let sql = format!(
1664 r#"CREATE SOURCE t
1665 WITH (connector = 'kinesis')
1666 FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecord', schema.location = 'file://{}')"#,
1667 proto_file.path().to_str().unwrap()
1668 );
1669 let frontend = LocalFrontend::new(Default::default()).await;
1670 frontend.run_sql(sql).await.unwrap();
1671
1672 let session = frontend.session_ref();
1673 let catalog_reader = session.env().catalog_reader().read_guard();
1674 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1675
1676 let (source, _) = catalog_reader
1678 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
1679 .unwrap();
1680 assert_eq!(source.name, "t");
1681
1682 let columns = GET_COLUMN_FROM_CATALOG(source);
1683
1684 let city_type = StructType::new(vec![
1685 ("address", DataType::Varchar),
1686 ("zipcode", DataType::Varchar),
1687 ])
1688 .into();
1690 let expected_columns = maplit::hashmap! {
1691 ROW_ID_COLUMN_NAME => DataType::Serial,
1692 "id" => DataType::Int32,
1693 "zipcode" => DataType::Int64,
1694 "rate" => DataType::Float32,
1695 "country" => StructType::new(
1696 vec![("address", DataType::Varchar),("city", city_type),("zipcode", DataType::Varchar)],
1697 )
1698 .into(),
1700 };
1701 assert_eq!(columns, expected_columns, "{columns:#?}");
1702 }
1703
1704 #[tokio::test]
1705 async fn test_create_mqtt_source_with_protobuf() {
1706 let proto_file = create_proto_file(PROTO_FILE_DATA);
1707 let sql = format!(
1708 r#"CREATE SOURCE t_mqtt
1709 WITH (
1710 connector = 'mqtt',
1711 url = 'mqtt://localhost:1883',
1712 topic = 'test_topic'
1713 )
1714 FORMAT PLAIN ENCODE PROTOBUF (
1715 message = '.test.TestRecord',
1716 schema.location = 'file://{}'
1717 )"#,
1718 proto_file.path().to_str().unwrap()
1719 );
1720 let frontend = LocalFrontend::new(Default::default()).await;
1721 frontend.run_sql(sql).await.unwrap();
1722
1723 let session = frontend.session_ref();
1724 let catalog_reader = session.env().catalog_reader().read_guard();
1725 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1726
1727 let (source, _) = catalog_reader
1728 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t_mqtt")
1729 .unwrap();
1730
1731 assert_eq!(source.name, "t_mqtt");
1732 assert_eq!(source.info.row_encode, EncodeType::Protobuf as i32);
1733 }
1734
1735 #[tokio::test]
1736 async fn test_create_posix_fs_source_requires_frontend_config() {
1737 let frontend = LocalFrontend::with_frontend_config(
1738 Default::default(),
1739 FrontendConfig {
1740 unsafe_enable_local_fs_connector: false,
1741 ..Default::default()
1742 },
1743 )
1744 .await;
1745 let err = frontend
1746 .run_sql(
1747 r#"CREATE SOURCE local_files (
1748 line VARCHAR
1749 ) WITH (
1750 connector = 'posix_fs',
1751 posix_fs.root = '/tmp',
1752 match_pattern = '*.csv'
1753 ) FORMAT PLAIN ENCODE CSV (without_header = 'true')"#
1754 .to_owned(),
1755 )
1756 .await
1757 .unwrap_err();
1758
1759 assert!(
1760 err.to_string()
1761 .contains("frontend.unsafe_enable_local_fs_connector = true"),
1762 "{err:?}"
1763 );
1764 }
1765
1766 #[tokio::test]
1767 async fn test_duplicate_props_options() {
1768 let proto_file = create_proto_file(PROTO_FILE_DATA);
1769 let sql = format!(
1770 r#"CREATE SOURCE t
1771 WITH (
1772 connector = 'kinesis',
1773 aws.region='user_test_topic',
1774 endpoint='172.10.1.1:9090,172.10.1.2:9090',
1775 aws.credentials.access_key_id = 'your_access_key_1',
1776 aws.credentials.secret_access_key = 'your_secret_key_1'
1777 )
1778 FORMAT PLAIN ENCODE PROTOBUF (
1779 message = '.test.TestRecord',
1780 aws.credentials.access_key_id = 'your_access_key_2',
1781 aws.credentials.secret_access_key = 'your_secret_key_2',
1782 schema.location = 'file://{}',
1783 )"#,
1784 proto_file.path().to_str().unwrap()
1785 );
1786 let frontend = LocalFrontend::new(Default::default()).await;
1787 frontend.run_sql(sql).await.unwrap();
1788
1789 let session = frontend.session_ref();
1790 let catalog_reader = session.env().catalog_reader().read_guard();
1791 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1792
1793 let (source, _) = catalog_reader
1795 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
1796 .unwrap();
1797 assert_eq!(source.name, "t");
1798
1799 assert_eq!(
1801 source
1802 .info
1803 .format_encode_options
1804 .get("aws.credentials.access_key_id")
1805 .unwrap(),
1806 "your_access_key_2"
1807 );
1808 assert_eq!(
1809 source
1810 .info
1811 .format_encode_options
1812 .get("aws.credentials.secret_access_key")
1813 .unwrap(),
1814 "your_secret_key_2"
1815 );
1816
1817 assert_eq!(
1819 source
1820 .with_properties
1821 .get("aws.credentials.access_key_id")
1822 .unwrap(),
1823 "your_access_key_1"
1824 );
1825 assert_eq!(
1826 source
1827 .with_properties
1828 .get("aws.credentials.secret_access_key")
1829 .unwrap(),
1830 "your_secret_key_1"
1831 );
1832
1833 assert!(!source.with_properties.contains_key("schema.location"));
1835 }
1836
1837 #[tokio::test]
1838 async fn test_multi_table_cdc_create_source_handler() {
1839 let sql =
1840 "CREATE SOURCE t2 WITH (connector = 'mysql-cdc') FORMAT PLAIN ENCODE JSON".to_owned();
1841 let frontend = LocalFrontend::new(Default::default()).await;
1842 let session = frontend.session_ref();
1843
1844 frontend
1845 .run_sql_with_session(session.clone(), sql)
1846 .await
1847 .unwrap();
1848 let catalog_reader = session.env().catalog_reader().read_guard();
1849 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1850
1851 let (source, _) = catalog_reader
1853 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t2")
1854 .unwrap();
1855 assert_eq!(source.name, "t2");
1856
1857 let columns = source
1858 .columns
1859 .iter()
1860 .map(|col| (col.name(), col.data_type().clone()))
1861 .collect::<Vec<(&str, DataType)>>();
1862
1863 expect_test::expect![[r#"
1864 [
1865 (
1866 "payload",
1867 Jsonb,
1868 ),
1869 (
1870 "_rw_offset",
1871 Varchar,
1872 ),
1873 (
1874 "_rw_table_name",
1875 Varchar,
1876 ),
1877 (
1878 "_row_id",
1879 Serial,
1880 ),
1881 ]
1882 "#]]
1883 .assert_debug_eq(&columns);
1884 }
1885
1886 #[tokio::test]
1887 async fn test_reject_zero_source_rate_limit_when_cdc_must_wait_for_offset() {
1888 let frontend = LocalFrontend::new(Default::default()).await;
1889 let session = frontend.session_ref();
1890
1891 for (source_name, connector) in [
1892 ("mysql_explicit_zero", "mysql-cdc"),
1893 ("sqlserver_explicit_zero", "sqlserver-cdc"),
1894 ] {
1895 let err = frontend
1896 .run_sql_with_session(
1897 session.clone(),
1898 format!(
1899 "CREATE SOURCE {source_name} WITH (connector = '{connector}', source_rate_limit = '0') FORMAT PLAIN ENCODE JSON"
1900 ),
1901 )
1902 .await
1903 .unwrap_err();
1904 let message = err.to_string();
1905 assert!(
1906 message.contains("source creation must wait for the initial CDC offset"),
1907 "{message}"
1908 );
1909 }
1910
1911 frontend
1912 .run_sql_with_session(session.clone(), "SET source_rate_limit TO 0;")
1913 .await
1914 .unwrap();
1915
1916 for (source_name, connector) in [
1917 ("mysql_session_zero", "mysql-cdc"),
1918 ("sqlserver_session_zero", "sqlserver-cdc"),
1919 ] {
1920 let err = frontend
1921 .run_sql_with_session(
1922 session.clone(),
1923 format!(
1924 "CREATE SOURCE {source_name} WITH (connector = '{connector}') FORMAT PLAIN ENCODE JSON"
1925 ),
1926 )
1927 .await
1928 .unwrap_err();
1929 assert!(
1930 err.to_string()
1931 .contains("source creation must wait for the initial CDC offset"),
1932 "{err}"
1933 );
1934 }
1935
1936 frontend
1937 .run_sql_with_session(
1938 session.clone(),
1939 "CREATE SOURCE mysql_positive_override WITH (connector = 'mysql-cdc', source_rate_limit = '1') FORMAT PLAIN ENCODE JSON",
1940 )
1941 .await
1942 .unwrap();
1943
1944 frontend
1945 .run_sql_with_session(
1946 session,
1947 "CREATE SOURCE postgres_zero WITH (connector = 'postgres-cdc') FORMAT PLAIN ENCODE JSON",
1948 )
1949 .await
1950 .unwrap();
1951 }
1952
1953 #[tokio::test]
1954 async fn test_source_addition_columns() {
1955 let sql =
1957 "CREATE SOURCE s (v1 int) include key as _rw_kafka_key with (connector = 'kafka') format plain encode json".to_owned();
1958 let frontend = LocalFrontend::new(Default::default()).await;
1959 frontend.run_sql(sql).await.unwrap();
1960 let session = frontend.session_ref();
1961 let catalog_reader = session.env().catalog_reader().read_guard();
1962 let (source, _) = catalog_reader
1963 .get_source_by_name(
1964 DEFAULT_DATABASE_NAME,
1965 SchemaPath::Name(DEFAULT_SCHEMA_NAME),
1966 "s",
1967 )
1968 .unwrap();
1969 assert_eq!(source.name, "s");
1970
1971 let columns = source
1972 .columns
1973 .iter()
1974 .map(|col| (col.name(), col.data_type().clone()))
1975 .collect::<Vec<(&str, DataType)>>();
1976
1977 expect_test::expect![[r#"
1978 [
1979 (
1980 "v1",
1981 Int32,
1982 ),
1983 (
1984 "_rw_kafka_key",
1985 Bytea,
1986 ),
1987 (
1988 "_rw_kafka_timestamp",
1989 Timestamptz,
1990 ),
1991 (
1992 "_rw_kafka_partition",
1993 Varchar,
1994 ),
1995 (
1996 "_rw_kafka_offset",
1997 Varchar,
1998 ),
1999 (
2000 "_row_id",
2001 Serial,
2002 ),
2003 ]
2004 "#]]
2005 .assert_debug_eq(&columns);
2006 drop(catalog_reader);
2007
2008 let sql =
2009 "CREATE SOURCE s_pulsar (v1 int) include header 'tenant' as pulsar_header with (connector = 'pulsar') format plain encode json".to_owned();
2010 frontend.run_sql(sql).await.unwrap();
2011 let catalog_reader = session.env().catalog_reader().read_guard();
2012 let (source, _) = catalog_reader
2013 .get_source_by_name(
2014 DEFAULT_DATABASE_NAME,
2015 SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2016 "s_pulsar",
2017 )
2018 .unwrap();
2019 assert_eq!(source.name, "s_pulsar");
2020
2021 let columns = source
2022 .columns
2023 .iter()
2024 .map(|col| (col.name(), col.data_type().clone()))
2025 .collect::<Vec<(&str, DataType)>>();
2026
2027 expect_test::expect![[r#"
2028 [
2029 (
2030 "v1",
2031 Int32,
2032 ),
2033 (
2034 "pulsar_header",
2035 Bytea,
2036 ),
2037 (
2038 "_row_id",
2039 Serial,
2040 ),
2041 ]
2042 "#]]
2043 .assert_debug_eq(&columns);
2044 drop(catalog_reader);
2045
2046 let sql =
2047 "CREATE SOURCE s_nats (v1 int) include header 'trace-id' as nats_header with (connector = 'nats') format plain encode json".to_owned();
2048 frontend.run_sql(sql).await.unwrap();
2049 let catalog_reader = session.env().catalog_reader().read_guard();
2050 let (source, _) = catalog_reader
2051 .get_source_by_name(
2052 DEFAULT_DATABASE_NAME,
2053 SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2054 "s_nats",
2055 )
2056 .unwrap();
2057 assert_eq!(source.name, "s_nats");
2058
2059 let columns = source
2060 .columns
2061 .iter()
2062 .map(|col| (col.name(), col.data_type().clone()))
2063 .collect::<Vec<(&str, DataType)>>();
2064
2065 expect_test::expect![[r#"
2066 [
2067 (
2068 "v1",
2069 Int32,
2070 ),
2071 (
2072 "nats_header",
2073 Bytea,
2074 ),
2075 (
2076 "_row_id",
2077 Serial,
2078 ),
2079 ]
2080 "#]]
2081 .assert_debug_eq(&columns);
2082 drop(catalog_reader);
2083
2084 let sql =
2086 "CREATE SOURCE s_nats_all (v1 int) include header as nats_all_headers with (connector = 'nats') format plain encode json".to_owned();
2087 frontend.run_sql(sql).await.unwrap();
2088 let catalog_reader = session.env().catalog_reader().read_guard();
2089 let (source, _) = catalog_reader
2090 .get_source_by_name(
2091 DEFAULT_DATABASE_NAME,
2092 SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2093 "s_nats_all",
2094 )
2095 .unwrap();
2096 assert_eq!(source.name, "s_nats_all");
2097 let all_headers_col = source
2098 .columns
2099 .iter()
2100 .find(|c| c.name() == "nats_all_headers")
2101 .expect("nats_all_headers column present");
2102 assert_eq!(
2103 all_headers_col.data_type(),
2104 &DataType::list(get_kafka_header_item_datatype()),
2105 );
2106 drop(catalog_reader);
2107
2108 let sql =
2109 "CREATE SOURCE s3 (v1 int) include timestamp 'header1' as header_col with (connector = 'kafka') format plain encode json".to_owned();
2110 match frontend.run_sql(sql).await {
2111 Err(e) => {
2112 assert_eq!(
2113 e.to_string(),
2114 "Protocol error: Only header column can have inner field, but got \"timestamp\""
2115 )
2116 }
2117 _ => unreachable!(),
2118 }
2119 }
2120
2121 #[tokio::test]
2125 async fn test_create_privilege_checked_before_schema_fetch() {
2126 let frontend = LocalFrontend::new(Default::default()).await;
2127 frontend
2128 .run_sql("CREATE USER no_create_user")
2129 .await
2130 .unwrap();
2131 let user_id = frontend
2132 .session_ref()
2133 .env()
2134 .user_info_reader()
2135 .read_guard()
2136 .get_user_by_name("no_create_user")
2137 .unwrap()
2138 .id;
2139 let user_session = frontend.session_user_ref(
2140 DEFAULT_DATABASE_NAME.to_owned(),
2141 "no_create_user".to_owned(),
2142 user_id,
2143 );
2144
2145 let with_and_format = "WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:1', topic = 't') \
2148 FORMAT PLAIN ENCODE AVRO (schema.registry = 'http://127.0.0.1:1')";
2149 let create_source = format!("CREATE SOURCE s {with_and_format}");
2150 let create_table = format!("CREATE TABLE t {with_and_format}");
2151
2152 for sql in [create_source, create_table] {
2153 let err = frontend
2154 .run_sql_with_session(user_session.clone(), sql)
2155 .await
2156 .unwrap_err();
2157 assert!(
2158 err.to_string()
2159 .contains("permission denied for schema \"public\": \"CREATE\""),
2160 "{err:?}"
2161 );
2162 }
2163 }
2164}