Skip to main content

risingwave_frontend/handler/
create_source.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, HashMap, HashSet};
16use std::rc::Rc;
17use std::sync::LazyLock;
18
19use anyhow::Context;
20use either::Either;
21use external_schema::debezium::extract_debezium_avro_table_pk_columns;
22use external_schema::nexmark::check_nexmark_schema;
23use itertools::Itertools;
24use maplit::{convert_args, hashmap, hashset};
25use pgwire::pg_response::{PgResponse, StatementType};
26use risingwave_common::array::arrow::{IcebergArrowConvert, arrow_schema_iceberg};
27use risingwave_common::bail_not_implemented;
28use risingwave_common::catalog::{
29    CdcTableDesc, ColumnCatalog, ColumnDesc, ColumnId, INITIAL_SOURCE_VERSION_ID,
30    KAFKA_TIMESTAMP_COLUMN_NAME, ROW_ID_COLUMN_NAME, TableId, debug_assert_column_ids_distinct,
31};
32use risingwave_common::license::Feature;
33use risingwave_common::secret::LocalSecretManager;
34use risingwave_common::system_param::reader::SystemParamsRead;
35use risingwave_common::types::DataType;
36use risingwave_common::util::iter_util::ZipEqFast;
37use risingwave_connector::parser::additional_columns::{
38    build_additional_column_desc, source_add_partition_offset_cols,
39};
40use risingwave_connector::parser::{
41    AvroParserConfig, DEBEZIUM_IGNORE_KEY, DebeziumAvroParserConfig, ProtobufParserConfig,
42    SchemaLocation, SpecificParserConfig, TimestamptzHandling,
43    fetch_json_schema_and_map_to_columns, fetch_pulsar_json_schema_and_map_to_columns,
44};
45use risingwave_connector::schema::AWS_GLUE_SCHEMA_ARN_KEY;
46use risingwave_connector::schema::pulsar_schema::{
47    PULSAR_SCHEMA_AUTH_TOKEN_KEY, PULSAR_SCHEMA_URL_KEY, PulsarSchemaConfig,
48};
49use risingwave_connector::schema::schema_registry::{
50    SCHEMA_REGISTRY_BACKOFF_DURATION_KEY, SCHEMA_REGISTRY_BACKOFF_FACTOR_KEY,
51    SCHEMA_REGISTRY_CA_PEM_PATH, SCHEMA_REGISTRY_MAX_DELAY_KEY, SCHEMA_REGISTRY_PASSWORD,
52    SCHEMA_REGISTRY_RETRIES_MAX_KEY, SCHEMA_REGISTRY_USERNAME, SchemaRegistryConfig,
53    name_strategy_from_str,
54};
55use risingwave_connector::source::cdc::{
56    CDC_BACKFILL_ENABLE_KEY, CDC_MONGODB_STRONG_SCHEMA_KEY,
57    CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY, CDC_SHARING_MODE_KEY, CDC_SNAPSHOT_BACKFILL,
58    CDC_SNAPSHOT_MODE_KEY, CDC_TRANSACTIONAL_KEY, CDC_WAIT_FOR_STREAMING_START_TIMEOUT,
59    CdcSourceDdlValidator, CdcSourceRuntimeFlags,
60};
61use risingwave_connector::source::datagen::DATAGEN_CONNECTOR;
62use risingwave_connector::source::iceberg::ICEBERG_CONNECTOR;
63use risingwave_connector::source::nexmark::source::{EventType, get_event_data_types_with_names};
64use risingwave_connector::source::test_source::TEST_CONNECTOR;
65pub use risingwave_connector::source::{
66    ADBC_SNOWFLAKE_CONNECTOR, UPSTREAM_SOURCE_KEY, WEBHOOK_CONNECTOR,
67};
68use risingwave_connector::source::{
69    AZBLOB_CONNECTOR, ConnectorProperties, GCS_CONNECTOR, GOOGLE_PUBSUB_CONNECTOR, KAFKA_CONNECTOR,
70    KINESIS_CONNECTOR, LEGACY_S3_CONNECTOR, MQTT_CONNECTOR, NATS_CONNECTOR, NEXMARK_CONNECTOR,
71    OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR, PULSAR_CONNECTOR,
72};
73use risingwave_connector::{AUTO_SCHEMA_CHANGE_KEY, WithPropertiesExt, dispatch_cdc_source_type};
74use risingwave_pb::catalog::connection_params::PbConnectionType;
75use risingwave_pb::catalog::{PbSchemaRegistryNameStrategy, StreamSourceInfo, WatermarkDesc};
76use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
77use risingwave_pb::plan_common::source_ingest_mode::{IngestMode, SourceIngestModeStreaming};
78use risingwave_pb::plan_common::{EncodeType, FormatType, SourceIngestMode};
79use risingwave_pb::stream_plan::PbStreamFragmentGraph;
80use risingwave_pb::telemetry::TelemetryDatabaseObject;
81use risingwave_sqlparser::ast::{
82    ColumnDef, ColumnOption, CreateSourceStatement, Encode, Format, FormatEncodeOptions,
83    ObjectName, SourceWatermark, SqlOptionValue, TableConstraint, Value, get_delimiter,
84};
85use risingwave_sqlparser::parser::{IncludeOption, IncludeOptionItem};
86use thiserror_ext::AsReport;
87
88use super::RwPgResponse;
89use crate::binder::Binder;
90use crate::catalog::CatalogError;
91use crate::catalog::root_catalog::SchemaPath;
92use crate::catalog::source_catalog::SourceCatalog;
93use crate::error::ErrorCode::{self, Deprecated, InvalidInputSyntax, NotSupported, ProtocolError};
94use crate::error::{Result, RwError};
95use crate::expr::{Expr, ExprRewriter, SessionTimezone};
96use crate::handler::HandlerArgs;
97use crate::handler::cdc::{
98    bind_cdc_pk_comparisons_externally, bind_cdc_table_schema, bind_cdc_table_schema_externally,
99    derive_connect_properties_for_cdc_table, not_null_check_for_cdc_table,
100    reject_pk_filtered_by_debezium_column_filter, sanity_check_for_table_on_cdc_source,
101};
102use crate::handler::create_table::{
103    ColumnIdGenerator, bind_pk_and_row_id_on_relation, bind_sql_column_constraints,
104    bind_sql_columns, bind_sql_pk_names, bind_table_constraints, check_cdc_source_select_privilege,
105};
106use crate::handler::util::{
107    SourceSchemaCompatExt, check_connector_match_connection_type, ensure_connection_type_allowed,
108    ensure_local_fs_connector_allowed,
109};
110use crate::optimizer::plan_node::generic::SourceNodeKind;
111use crate::optimizer::plan_node::{BackfillType, LogicalSource, ToStream, ToStreamContext};
112use crate::session::SessionImpl;
113use crate::session::current::notice_to_user;
114use crate::utils::{
115    OverwriteOptions, resolve_connection_ref_and_secret_ref, resolve_privatelink_in_with_option,
116    resolve_secret_ref_in_with_options, resolve_source_ingest_mode_in_with_option,
117};
118use crate::{OptimizerContext, WithOptions, WithOptionsSecResolved, bind_data_type, build_graph};
119
120mod external_schema;
121pub use external_schema::{
122    bind_columns_from_source, get_schema_location, schema_has_schema_registry,
123};
124use external_schema::{format_to_prost, row_encode_to_prost};
125mod validate;
126pub use validate::validate_compatibility;
127use validate::{SOURCE_ALLOWED_CONNECTION_CONNECTOR, SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY};
128mod additional_column;
129use additional_column::check_and_add_timestamp_column;
130pub use additional_column::handle_addition_columns;
131use risingwave_common::catalog::ICEBERG_SOURCE_PREFIX;
132use risingwave_common::id::SourceId;
133
134use crate::stream_fragmenter::GraphJobType;
135
136fn non_generated_sql_columns(columns: &[ColumnDef]) -> Vec<ColumnDef> {
137    columns
138        .iter()
139        .filter(|c| !c.is_generated())
140        .cloned()
141        .collect()
142}
143
144fn try_consume_string_from_options(
145    format_encode_options: &mut BTreeMap<String, String>,
146    key: &str,
147) -> Option<String> {
148    format_encode_options.remove(key)
149}
150
151fn try_consume_schema_registry_config_from_options(
152    format_encode_options: &mut BTreeMap<String, String>,
153) {
154    [
155        SCHEMA_REGISTRY_USERNAME,
156        SCHEMA_REGISTRY_PASSWORD,
157        SCHEMA_REGISTRY_CA_PEM_PATH,
158        SCHEMA_REGISTRY_MAX_DELAY_KEY,
159        SCHEMA_REGISTRY_BACKOFF_DURATION_KEY,
160        SCHEMA_REGISTRY_BACKOFF_FACTOR_KEY,
161        SCHEMA_REGISTRY_RETRIES_MAX_KEY,
162    ]
163    .iter()
164    .for_each(|key| {
165        try_consume_string_from_options(format_encode_options, key);
166    });
167}
168
169/// Returns whether the Pulsar schema is used, with `schema.pulsar.url` set directly or as a secret.
170pub(crate) fn uses_pulsar_schema<V>(
171    format_encode_options: &BTreeMap<String, String>,
172    format_encode_secret_refs: &BTreeMap<String, V>,
173) -> bool {
174    format_encode_options.contains_key(PULSAR_SCHEMA_URL_KEY)
175        || format_encode_secret_refs.contains_key(PULSAR_SCHEMA_URL_KEY)
176}
177
178fn try_consume_pulsar_schema_config_from_options(
179    format_encode_options: &mut BTreeMap<String, String>,
180) {
181    for key in [PULSAR_SCHEMA_URL_KEY, PULSAR_SCHEMA_AUTH_TOKEN_KEY] {
182        try_consume_string_from_options(format_encode_options, key);
183    }
184}
185
186fn consume_string_from_options(
187    format_encode_options: &mut BTreeMap<String, String>,
188    key: &str,
189) -> Result<String> {
190    try_consume_string_from_options(format_encode_options, key).ok_or(RwError::from(ProtocolError(
191        format!("missing field {} in options", key),
192    )))
193}
194
195fn consume_aws_config_from_options(format_encode_options: &mut BTreeMap<String, String>) {
196    format_encode_options.retain(|key, _| !key.starts_with("aws."))
197}
198
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
200pub enum CreateSourceType {
201    SharedCdc,
202    /// e.g., shared Kafka source
203    SharedNonCdc,
204    NonShared,
205    /// create table with connector
206    Table,
207}
208
209impl CreateSourceType {
210    /// Note: shouldn't be used for `ALTER SOURCE`, since session variables should not affect existing source. We should respect the original type instead.
211    pub fn for_newly_created(
212        session: &SessionImpl,
213        with_properties: &impl WithPropertiesExt,
214    ) -> Self {
215        if with_properties.is_shareable_cdc_connector() {
216            CreateSourceType::SharedCdc
217        } else if with_properties.is_shareable_non_cdc_connector()
218            && session
219                .env()
220                .streaming_config()
221                .developer
222                .enable_shared_source
223            && session.config().streaming_use_shared_source()
224        {
225            CreateSourceType::SharedNonCdc
226        } else {
227            CreateSourceType::NonShared
228        }
229    }
230
231    pub fn for_replace(catalog: &SourceCatalog) -> Self {
232        if !catalog.info.is_shared() {
233            CreateSourceType::NonShared
234        } else if catalog.with_properties.is_shareable_cdc_connector() {
235            CreateSourceType::SharedCdc
236        } else {
237            CreateSourceType::SharedNonCdc
238        }
239    }
240
241    pub fn is_shared(&self) -> bool {
242        matches!(
243            self,
244            CreateSourceType::SharedCdc | CreateSourceType::SharedNonCdc
245        )
246    }
247}
248
249/// Bind columns from both source and sql defined.
250pub(crate) fn bind_all_columns(
251    format_encode: &FormatEncodeOptions,
252    cols_from_source: Option<Vec<ColumnCatalog>>,
253    cols_from_sql: Vec<ColumnCatalog>,
254    col_defs_from_sql: &[ColumnDef],
255    wildcard_idx: Option<usize>,
256    sql_column_strategy: SqlColumnStrategy,
257) -> Result<Vec<ColumnCatalog>> {
258    if let Some(cols_from_source) = cols_from_source {
259        // Need to check `col_defs` to see if a column is generated, as we haven't bind the
260        // `GeneratedColumnDesc` in `ColumnCatalog` yet.
261        let generated_cols_from_sql = cols_from_sql
262            .iter()
263            .filter(|c| {
264                col_defs_from_sql
265                    .iter()
266                    .find(|d| d.name.real_value() == c.name())
267                    .unwrap()
268                    .is_generated()
269            })
270            .cloned()
271            .collect_vec();
272
273        match sql_column_strategy {
274            // Ignore `cols_from_source`, follow `cols_from_sql` without checking.
275            SqlColumnStrategy::FollowUnchecked => {
276                assert!(
277                    wildcard_idx.is_none(),
278                    "wildcard still exists while strategy is Follows, not correctly purified?"
279                );
280                return Ok(cols_from_sql);
281            }
282
283            // Will merge `generated_cols_from_sql` into `cols_from_source`.
284            SqlColumnStrategy::Ignore => {}
285
286            SqlColumnStrategy::FollowChecked => {
287                let has_regular_cols_from_sql =
288                    generated_cols_from_sql.len() != cols_from_sql.len();
289
290                if has_regular_cols_from_sql {
291                    if wildcard_idx.is_some() {
292                        // (*, normal_column INT)
293                        return Err(RwError::from(NotSupported(
294                            "When there's a wildcard (\"*\"), \
295                             only generated columns are allowed in user-defined schema from SQL"
296                                .to_owned(),
297                            "Remove the non-generated columns".to_owned(),
298                        )));
299                    } else {
300                        // (normal_column INT)
301                        // Follow `cols_from_sql` with name & type checking.
302                        for col in &cols_from_sql {
303                            if generated_cols_from_sql.contains(col) {
304                                continue;
305                            }
306                            let Some(col_from_source) =
307                                cols_from_source.iter().find(|c| c.name() == col.name())
308                            else {
309                                return Err(RwError::from(ProtocolError(format!(
310                                    "Column \"{}\" is defined in SQL but not found in the source",
311                                    col.name()
312                                ))));
313                            };
314
315                            if col_from_source.data_type() != col.data_type() {
316                                return Err(RwError::from(ProtocolError(format!(
317                                    "Data type mismatch for column \"{}\". \
318                                     Defined in SQL as \"{}\", but found in the source as \"{}\"",
319                                    col.name(),
320                                    col.data_type(),
321                                    col_from_source.data_type()
322                                ))));
323                            }
324                        }
325                        return Ok(cols_from_sql);
326                    }
327                } else {
328                    if wildcard_idx.is_some() {
329                        // (*)
330                        // (*, generated_column INT)
331                        // Good, expand the wildcard later.
332                    } else {
333                        // ()
334                        // (generated_column INT)
335                        // Interpreted as if there's a wildcard for backward compatibility.
336                        // TODO: the behavior is not that consistent, making it impossible to ingest no
337                        //       columns from the source (though not useful in practice). Currently we
338                        //       just notice the user but we may want to interpret it as empty columns
339                        //       in the future.
340                        notice_to_user("\
341                            Neither wildcard (\"*\") nor regular (non-generated) columns appear in the user-defined schema from SQL. \
342                            For backward compatibility, all columns from the source will be included at the beginning. \
343                            For clarity, consider adding a wildcard (\"*\") to indicate where the columns from the source should be included, \
344                            or specifying the columns you want to include from the source.
345                        ");
346                    }
347                }
348            }
349        }
350
351        // In some cases the wildcard may be absent:
352        // - plan based on a purified SQL
353        // - interpret `()` as `(*)` for backward compatibility (see notice above)
354        // Default to 0 to expand the wildcard at the beginning.
355        let wildcard_idx = wildcard_idx.unwrap_or(0).min(generated_cols_from_sql.len());
356
357        // Merge `generated_cols_from_sql` with `cols_from_source`.
358        let mut merged_cols = generated_cols_from_sql;
359        let merged_cols_r = merged_cols.split_off(wildcard_idx);
360        merged_cols.extend(cols_from_source);
361        merged_cols.extend(merged_cols_r);
362
363        Ok(merged_cols)
364    } else {
365        if wildcard_idx.is_some() {
366            return Err(RwError::from(NotSupported(
367                "Wildcard in user-defined schema is only allowed when there exists columns from external schema".to_owned(),
368                "Remove the wildcard or use a source with external schema".to_owned(),
369            )));
370        }
371        let non_generated_sql_defined_columns = non_generated_sql_columns(col_defs_from_sql);
372
373        match (&format_encode.format, &format_encode.row_encode) {
374            (Format::DebeziumMongo, Encode::Json) => {
375                let strong_schema = format_encode
376                    .row_options
377                    .iter()
378                    .find(|k| k.name.real_value().to_lowercase() == CDC_MONGODB_STRONG_SCHEMA_KEY)
379                    .map(|k| matches!(k.value, SqlOptionValue::Value(Value::Boolean(true))))
380                    .unwrap_or(false);
381
382                // strong schema requires a '_id' column at the first position with a specific type
383                if strong_schema {
384                    let (_, id_column) = non_generated_sql_defined_columns
385                        .iter()
386                        .enumerate()
387                        .find(|(idx, col)| *idx == 0 && col.name.real_value() == "_id")
388                        .ok_or_else(|| {
389                            RwError::from(ProtocolError(
390                                "The `_id` column of the source with row format DebeziumMongoJson must be defined as the first column in SQL".to_owned(),
391                            ))
392                        })?;
393
394                    let id_data_type = bind_data_type(id_column.data_type.as_ref().unwrap())?;
395                    if !matches!(
396                        id_data_type,
397                        DataType::Varchar | DataType::Int32 | DataType::Int64 | DataType::Jsonb
398                    ) {
399                        return Err(RwError::from(ProtocolError(
400                            "the `_id` column of the source with row format DebeziumMongoJson must be [Jsonb | Varchar | Int32 | Int64]".to_owned(),
401                        )));
402                    }
403
404                    let mut columns = Vec::with_capacity(non_generated_sql_defined_columns.len());
405                    columns.push(
406                        // id column
407                        ColumnCatalog {
408                            column_desc: ColumnDesc::named("_id", 0.into(), id_data_type),
409                            is_hidden: false,
410                        },
411                    );
412
413                    // bind rest of the columns
414                    for (idx, col) in non_generated_sql_defined_columns
415                        .into_iter()
416                        // skip the first column
417                        .skip(1)
418                        .enumerate()
419                    {
420                        columns.push(ColumnCatalog {
421                            column_desc: ColumnDesc::named(
422                                col.name.real_value(),
423                                (idx as i32).into(),
424                                bind_data_type(col.data_type.as_ref().unwrap())?,
425                            ),
426                            is_hidden: false,
427                        });
428                    }
429
430                    return Ok(columns);
431                }
432
433                let mut columns = vec![
434                    ColumnCatalog {
435                        column_desc: ColumnDesc::named("_id", 0.into(), DataType::Varchar),
436                        is_hidden: false,
437                    },
438                    ColumnCatalog {
439                        column_desc: ColumnDesc::named("payload", 0.into(), DataType::Jsonb),
440                        is_hidden: false,
441                    },
442                ];
443
444                if non_generated_sql_defined_columns.len() != 2
445                    || non_generated_sql_defined_columns[0].name.real_value() != columns[0].name()
446                    || non_generated_sql_defined_columns[1].name.real_value() != columns[1].name()
447                {
448                    return Err(RwError::from(ProtocolError(
449                        "the not generated columns of the source with row format DebeziumMongoJson
450        must be (_id [Jsonb | Varchar | Int32 | Int64], payload jsonb)."
451                            .to_owned(),
452                    )));
453                }
454                // ok to unwrap since it was checked at `bind_sql_columns`
455                let key_data_type = bind_data_type(
456                    non_generated_sql_defined_columns[0]
457                        .data_type
458                        .as_ref()
459                        .unwrap(),
460                )?;
461                match key_data_type {
462                    DataType::Jsonb | DataType::Varchar | DataType::Int32 | DataType::Int64 => {
463                        columns[0].column_desc.data_type = key_data_type;
464                    }
465                    _ => {
466                        return Err(RwError::from(ProtocolError(
467                            "the `_id` column of the source with row format DebeziumMongoJson
468        must be [Jsonb | Varchar | Int32 | Int64]"
469                                .to_owned(),
470                        )));
471                    }
472                }
473
474                // ok to unwrap since it was checked at `bind_sql_columns`
475                let value_data_type = bind_data_type(
476                    non_generated_sql_defined_columns[1]
477                        .data_type
478                        .as_ref()
479                        .unwrap(),
480                )?;
481                if !matches!(value_data_type, DataType::Jsonb) {
482                    return Err(RwError::from(ProtocolError(
483                        "the `payload` column of the source with row format DebeziumMongoJson
484        must be Jsonb datatype"
485                            .to_owned(),
486                    )));
487                }
488                Ok(columns)
489            }
490            (Format::Plain, Encode::Bytes) => {
491                let err = Err(RwError::from(ProtocolError(
492                    "ENCODE BYTES only accepts one BYTEA type column".to_owned(),
493                )));
494                if non_generated_sql_defined_columns.len() == 1 {
495                    // ok to unwrap `data_type`` since it was checked at `bind_sql_columns`
496                    let col_data_type = bind_data_type(
497                        non_generated_sql_defined_columns[0]
498                            .data_type
499                            .as_ref()
500                            .unwrap(),
501                    )?;
502                    if col_data_type == DataType::Bytea {
503                        Ok(cols_from_sql)
504                    } else {
505                        err
506                    }
507                } else {
508                    err
509                }
510            }
511            (_, _) => Ok(cols_from_sql),
512        }
513    }
514}
515
516/// TODO: perhaps put the hint in notice is better. The error message format might be not that reliable.
517fn hint_format_encode(format_encode: &FormatEncodeOptions) -> String {
518    format!(
519        r#"Hint: For FORMAT {0} ENCODE {1}, INCLUDE KEY must be specified and the key column must be used as primary key.
520example:
521    CREATE TABLE <table_name> ( PRIMARY KEY ([rw_key | <key_name>]) )
522    INCLUDE KEY [AS <key_name>]
523    WITH (...)
524    FORMAT {0} ENCODE {1}{2}
525"#,
526        format_encode.format,
527        format_encode.row_encode,
528        if format_encode.row_encode == Encode::Json || format_encode.row_encode == Encode::Bytes {
529            "".to_owned()
530        } else {
531            " (...)".to_owned()
532        }
533    )
534}
535
536/// Bind column from source. Add key column to table columns if necessary.
537/// Return `pk_names`.
538pub(crate) async fn bind_source_pk(
539    format_encode: &FormatEncodeOptions,
540    source_info: &StreamSourceInfo,
541    columns: &mut [ColumnCatalog],
542    sql_defined_pk_names: Vec<String>,
543    with_properties: &WithOptionsSecResolved,
544) -> Result<Vec<String>> {
545    if with_properties.is_iceberg_update_source() {
546        if sql_defined_pk_names.is_empty() {
547            return Err(InvalidInputSyntax(
548                "Iceberg streaming_updates requires a SQL PRIMARY KEY for retracting ingestion"
549                    .to_owned(),
550            )
551            .into());
552        }
553        return Ok(sql_defined_pk_names);
554    }
555    let sql_defined_pk = !sql_defined_pk_names.is_empty();
556    let include_key_column_name: Option<String> = {
557        // iter columns to check if contains additional columns from key part
558        // return the key column names if exists
559        columns.iter().find_map(|catalog| {
560            if matches!(
561                catalog.column_desc.additional_column.column_type,
562                Some(AdditionalColumnType::Key(_))
563            ) {
564                Some(catalog.name().to_owned())
565            } else {
566                None
567            }
568        })
569    };
570    let additional_column_names = columns
571        .iter()
572        .filter_map(|col| {
573            if col.column_desc.additional_column.column_type.is_some() {
574                Some(col.name().to_owned())
575            } else {
576                None
577            }
578        })
579        .collect_vec();
580
581    let res = match (&format_encode.format, &format_encode.row_encode) {
582        (Format::Native, Encode::Native) | (Format::None, Encode::None) | (Format::Plain, _) => {
583            sql_defined_pk_names
584        }
585
586        // For all Upsert formats, we only accept one and only key column as primary key.
587        // Additional KEY columns must be set in this case and must be primary key.
588        (Format::Upsert, Encode::Json | Encode::Avro | Encode::Protobuf) => {
589            if let Some(ref key_column_name) = include_key_column_name
590                && sql_defined_pk
591            {
592                // pk is set. check if it's valid
593
594                // the column name have been converted to real value in `handle_addition_columns`
595                // so we don't ignore ascii case here
596                if sql_defined_pk_names.len() != 1
597                    || !key_column_name.eq(sql_defined_pk_names[0].as_str())
598                {
599                    return Err(RwError::from(ProtocolError(format!(
600                        "Only \"{}\" can be used as primary key\n\n{}",
601                        key_column_name,
602                        hint_format_encode(format_encode)
603                    ))));
604                }
605                sql_defined_pk_names
606            } else {
607                // pk not set, or even key not included
608                return if let Some(include_key_column_name) = include_key_column_name {
609                    Err(RwError::from(ProtocolError(format!(
610                        "Primary key must be specified to {}\n\n{}",
611                        include_key_column_name,
612                        hint_format_encode(format_encode)
613                    ))))
614                } else {
615                    Err(RwError::from(ProtocolError(format!(
616                        "INCLUDE KEY clause not set\n\n{}",
617                        hint_format_encode(format_encode)
618                    ))))
619                };
620            }
621        }
622
623        (Format::Debezium, Encode::Json) => {
624            if !additional_column_names.is_empty() {
625                return Err(RwError::from(ProtocolError(format!(
626                    "FORMAT DEBEZIUM forbids additional columns, but got {:?}",
627                    additional_column_names
628                ))));
629            }
630            if !sql_defined_pk {
631                return Err(RwError::from(ProtocolError(
632                    "Primary key must be specified when creating source with FORMAT DEBEZIUM."
633                        .to_owned(),
634                )));
635            }
636            sql_defined_pk_names
637        }
638        (Format::Debezium, Encode::Avro) => {
639            if !additional_column_names.is_empty() {
640                return Err(RwError::from(ProtocolError(format!(
641                    "FORMAT DEBEZIUM forbids additional columns, but got {:?}",
642                    additional_column_names
643                ))));
644            }
645            if sql_defined_pk {
646                sql_defined_pk_names
647            } else {
648                let pk_names =
649                    extract_debezium_avro_table_pk_columns(source_info, with_properties).await?;
650                // extract pk(s) from schema registry
651                for pk_name in &pk_names {
652                    columns
653                        .iter()
654                        .find(|c: &&ColumnCatalog| c.name().eq(pk_name))
655                        .ok_or_else(|| {
656                            RwError::from(ProtocolError(format!(
657                                "avro's key column {} not exists in avro's row schema",
658                                pk_name
659                            )))
660                        })?;
661                }
662                pk_names
663            }
664        }
665        (Format::DebeziumMongo, Encode::Json) => {
666            if sql_defined_pk {
667                sql_defined_pk_names
668            } else {
669                vec!["_id".to_owned()]
670            }
671        }
672
673        (Format::Maxwell, Encode::Json) => {
674            if !additional_column_names.is_empty() {
675                return Err(RwError::from(ProtocolError(format!(
676                    "FORMAT MAXWELL forbids additional columns, but got {:?}",
677                    additional_column_names
678                ))));
679            }
680            if !sql_defined_pk {
681                return Err(RwError::from(ProtocolError(
682    "Primary key must be specified when creating source with FORMAT MAXWELL ENCODE JSON.".to_owned(),
683    )));
684            }
685            sql_defined_pk_names
686        }
687
688        (Format::Canal, Encode::Json) => {
689            if !additional_column_names.is_empty() {
690                return Err(RwError::from(ProtocolError(format!(
691                    "FORMAT CANAL forbids additional columns, but got {:?}",
692                    additional_column_names
693                ))));
694            }
695            if !sql_defined_pk {
696                return Err(RwError::from(ProtocolError(
697    "Primary key must be specified when creating source with FORMAT CANAL ENCODE JSON.".to_owned(),
698    )));
699            }
700            sql_defined_pk_names
701        }
702        (format, encoding) => {
703            return Err(RwError::from(ProtocolError(format!(
704                "Unknown combination {:?} {:?}",
705                format, encoding
706            ))));
707        }
708    };
709    Ok(res)
710}
711
712pub(super) fn bind_source_watermark(
713    session: &SessionImpl,
714    name: String,
715    source_watermarks: Vec<SourceWatermark>,
716    column_catalogs: &[ColumnCatalog],
717) -> Result<Vec<WatermarkDesc>> {
718    let mut binder = Binder::new_for_ddl(session);
719    binder.bind_columns_to_context(name.clone(), column_catalogs)?;
720
721    let mut session_tz = SessionTimezone::new(session.config().timezone());
722
723    let watermark_descs = source_watermarks
724        .into_iter()
725        .map(|source_watermark| {
726            let col_name = source_watermark.column.real_value();
727            let watermark_idx = binder.get_column_binding_index(name.clone(), &col_name)?;
728
729            let expr = binder.bind_expr(&source_watermark.expr)?;
730            // Apply session timezone rewrite so that timestamptz +/- interval operations
731            // are transformed into the timezone-aware variant (e.g. `subtract_with_time_zone`).
732            // Without this, intervals containing days/months would fail at runtime.
733            let expr = session_tz.rewrite_expr(expr);
734            let watermark_col_type = column_catalogs[watermark_idx].data_type();
735            let watermark_expr_type = &expr.return_type();
736            if watermark_col_type != watermark_expr_type {
737                Err(RwError::from(ErrorCode::BindError(
738                    format!("The return value type of the watermark expression must be identical to the watermark column data type. Current data type of watermark return value: `{}`, column `{}`",watermark_expr_type, watermark_col_type),
739                )))
740            } else {
741                let expr_proto = expr.to_expr_proto();
742                Ok::<_, RwError>(WatermarkDesc {
743                    watermark_idx: watermark_idx as u32,
744                    expr: Some(expr_proto),
745                    with_ttl: source_watermark.with_ttl,
746                })
747            }
748        })
749        .try_collect()?;
750    Ok(watermark_descs)
751}
752
753/// Performs early stage checking in frontend to see if the schema of the given `columns` is
754/// compatible with the connector extracted from the properties.
755///
756/// One should only call this function after all properties of all columns are resolved, like
757/// generated column descriptors.
758pub(super) fn check_format_encode(
759    props: &WithOptionsSecResolved,
760    row_id_index: Option<usize>,
761    columns: &[ColumnCatalog],
762) -> Result<()> {
763    let Some(connector) = props.get_connector() else {
764        return Ok(());
765    };
766
767    if connector == NEXMARK_CONNECTOR {
768        check_nexmark_schema(props, row_id_index, columns)
769    } else {
770        Ok(())
771    }
772}
773
774pub fn bind_connector_props(
775    handler_args: &HandlerArgs,
776    format_encode: &FormatEncodeOptions,
777    is_create_source: bool,
778) -> Result<(WithOptions, SourceIngestMode)> {
779    let mut with_properties = handler_args.with_options.clone().into_connector_props();
780    // The planner and executors decide the ingestion mode from plaintext options only, while
781    // connector property extraction would also read a secret value.
782    if with_properties.is_iceberg_connector()
783        && with_properties
784            .secret_ref()
785            .contains_key(risingwave_connector::source::iceberg::STREAMING_UPDATES_KEY)
786    {
787        return Err(RwError::from(ErrorCode::InvalidParameterValue(
788            "streaming_updates cannot be provided as a SECRET; use a plaintext 'true' or 'false'"
789                .to_owned(),
790        )));
791    }
792    let (_, secret_refs) =
793        resolve_secret_ref_in_with_options(with_properties.clone(), &handler_args.session)?
794            .into_parts();
795    validate_compatibility(format_encode, &mut with_properties, &secret_refs)?;
796    let ingest_mode = {
797        let ingest_mode = resolve_source_ingest_mode_in_with_option(&mut with_properties)?;
798        if is_create_source && ingest_mode.is_some() {
799            return Err(RwError::from(ProtocolError(
800                "`refresh_mode` only supported for CREATE TABLE".to_owned(),
801            )));
802        }
803
804        ingest_mode.unwrap_or(SourceIngestMode {
805            ingest_mode: Some(IngestMode::Streaming(SourceIngestModeStreaming {})),
806        })
807    };
808
809    let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
810
811    if !is_create_source && with_properties.is_shareable_cdc_connector() {
812        return Err(RwError::from(ProtocolError(format!(
813            "directly creating a CDC table for connector {} is no longer supported; \
814             please `CREATE SOURCE` to create a shared CDC source first, \
815             then `CREATE TABLE ... FROM <source> TABLE '<database>.<table>'`",
816            with_properties.get_connector().unwrap(),
817        ))));
818    }
819    if with_properties.is_cdc_connector() {
820        if with_properties.contains_key(CDC_TRANSACTIONAL_KEY)
821            || with_properties
822                .secret_ref()
823                .contains_key(CDC_TRANSACTIONAL_KEY)
824        {
825            return Err(RwError::from(ErrorCode::InvalidInputSyntax(format!(
826                "'{}' cannot be set: RisingWave enables transaction metadata itself for the connectors that support it",
827                CDC_TRANSACTIONAL_KEY
828            ))));
829        }
830        if with_properties.contains_key(CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY)
831            || with_properties
832                .secret_ref()
833                .contains_key(CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY)
834        {
835            return Err(RwError::from(ErrorCode::InvalidInputSyntax(format!(
836                "'{}' cannot be set: RisingWave sets it itself for the connectors that commit offsets to the upstream database",
837                CDC_NEEDS_OFFSET_COMMIT_ON_CHECKPOINT_KEY
838            ))));
839        }
840        if with_properties
841            .secret_ref()
842            .contains_key(AUTO_SCHEMA_CHANGE_KEY)
843        {
844            return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
845                "'{}' cannot be provided as a SECRET; use a plaintext 'true' or 'false'",
846                AUTO_SCHEMA_CHANGE_KEY
847            ))));
848        }
849        if let Some(value) = with_properties.get(AUTO_SCHEMA_CHANGE_KEY)
850            && value.parse::<bool>().map_err(|_| {
851                ErrorCode::InvalidInputSyntax(format!(
852                    "invalid value of '{}' option",
853                    AUTO_SCHEMA_CHANGE_KEY
854                ))
855            })?
856        {
857            if !dispatch_cdc_source_type!(
858                with_properties.cdc_source_type(),
859                T,
860                T::CAN_AUTO_SCHEMA_CHANGE,
861                false
862            ) {
863                return Err(ErrorCode::NotSupported(
864                    format!(
865                        "auto schema change for connector {}",
866                        with_properties.get_connector().unwrap()
867                    ),
868                    format!(
869                        "Remove the '{}' option or set it to false",
870                        AUTO_SCHEMA_CHANGE_KEY
871                    ),
872                )
873                .into());
874            }
875            Feature::CdcAutoSchemaChange.check_available()?;
876        }
877    }
878    if is_create_source && create_cdc_source_job {
879        // set connector to backfill mode
880        with_properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), CDC_SNAPSHOT_BACKFILL.into());
881        // enable cdc sharing mode, which will capture all tables in the given `database.name`
882        with_properties.insert(CDC_SHARING_MODE_KEY.into(), "true".into());
883        // enable transactional cdc
884        if with_properties.enable_transaction_metadata() {
885            with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
886        }
887        // Only set CDC_WAIT_FOR_STREAMING_START_TIMEOUT if not already specified by user.
888        if !with_properties.contains_key(CDC_WAIT_FOR_STREAMING_START_TIMEOUT) {
889            with_properties.insert(
890                CDC_WAIT_FOR_STREAMING_START_TIMEOUT.into(),
891                handler_args
892                    .session
893                    .config()
894                    .cdc_source_wait_streaming_start_timeout()
895                    .to_string(),
896            );
897        }
898    }
899    Ok((with_properties, ingest_mode))
900}
901
902/// When the schema can be inferred from external system (like schema registry),
903/// how to handle the regular columns (i.e., non-generated) defined in SQL?
904pub enum SqlColumnStrategy {
905    /// Follow all columns defined in SQL, ignore the columns from external system.
906    /// This ensures that no accidental side effect will change the schema.
907    ///
908    /// This is the behavior when re-planning the target table of `SINK INTO`.
909    FollowUnchecked,
910
911    /// Follow all column defined in SQL, check the columns from external system with name & type.
912    /// This ensures that no accidental side effect will change the schema.
913    ///
914    /// This is the behavior when creating a new table or source with a resolvable schema;
915    /// adding, or dropping columns on that table.
916    // TODO(purify): `ALTER SOURCE` currently has its own code path and does not check.
917    FollowChecked,
918
919    /// Merge the generated columns defined in SQL and columns from external system. If there
920    /// are also regular columns defined in SQL, ignore silently.
921    ///
922    /// This is the behavior when `REFRESH SCHEMA` atop the purified SQL.
923    Ignore,
924}
925
926/// Reject user-declared `VARIANT` columns (including nested) on binding paths whose ingest
927/// pipeline cannot produce variant values. `context` names the path in the error message.
928pub(crate) fn reject_variant_columns(columns: &[ColumnCatalog], context: &str) -> Result<()> {
929    if let Some(col) = columns.iter().find(|c| c.data_type().contains_variant()) {
930        return Err(RwError::from(NotSupported(
931            format!(
932                "VARIANT column \"{}\" is not supported {context} yet",
933                col.name()
934            ),
935            format!("VARIANT columns are not supported {context} yet"),
936        )));
937    }
938    Ok(())
939}
940
941/// Reject user-declared `VARIANT` columns when the `FORMAT ... ENCODE ...` is handled by a
942/// row-encode parser or native generator, none of which can produce variant values.
943///
944/// Not gated: connector-native schemas (`ENCODE NONE`, e.g. iceberg) infer their own schema, and
945/// `ENCODE PARQUET` reads variant columns via the Parquet Variant extension type.
946fn reject_variant_columns_for_unsupported_encoding(
947    format_encode: &FormatEncodeOptions,
948    columns_from_sql: &[ColumnCatalog],
949) -> Result<()> {
950    if matches!(
951        (&format_encode.format, &format_encode.row_encode),
952        (Format::None, Encode::None) | (_, Encode::Parquet)
953    ) {
954        return Ok(());
955    }
956    reject_variant_columns(columns_from_sql, "for this source encoding")
957}
958
959/// Entrypoint for binding source connector.
960/// Common logic shared by `CREATE SOURCE` and `CREATE TABLE`.
961#[expect(clippy::too_many_arguments)]
962pub async fn bind_create_source_or_table_with_connector(
963    handler_args: HandlerArgs,
964    full_name: ObjectName,
965    format_encode: FormatEncodeOptions,
966    with_properties: WithOptions,
967    sql_columns_defs: &[ColumnDef],
968    constraints: Vec<TableConstraint>,
969    wildcard_idx: Option<usize>,
970    source_watermarks: Vec<SourceWatermark>,
971    columns_from_resolve_source: Option<Vec<ColumnCatalog>>,
972    source_info: StreamSourceInfo,
973    include_column_options: IncludeOption,
974    col_id_gen: &mut ColumnIdGenerator,
975    create_source_type: CreateSourceType,
976    source_rate_limit: Option<u32>,
977    sql_column_strategy: SqlColumnStrategy,
978    ingest_mode: SourceIngestMode,
979) -> Result<SourceCatalog> {
980    let session = &handler_args.session;
981    let db_name: &str = &session.database();
982    let (schema_name, source_name) = Binder::resolve_schema_qualified_name(db_name, &full_name)?;
983    let (database_id, schema_id) =
984        session.get_database_and_schema_id_for_create(schema_name.clone())?;
985
986    let is_create_source = create_source_type != CreateSourceType::Table;
987
988    if is_create_source {
989        // reject refreshable batch source
990        if with_properties.is_batch_connector() {
991            return Err(ErrorCode::BindError(
992            "can't CREATE SOURCE with refreshable batch connector\n\nHint: use CREATE TABLE instead"
993                .to_owned(),
994        )
995        .into());
996        }
997
998        match format_encode.format {
999            // reject unsupported formats for CREATE SOURCE
1000            Format::Debezium | Format::DebeziumMongo | Format::Maxwell | Format::Canal => {
1001                return Err(ErrorCode::BindError(format!(
1002                    "can't CREATE SOURCE with FORMAT {}.\n\nHint: use CREATE TABLE instead\n\n{}",
1003                    format_encode.format,
1004                    hint_format_encode(&format_encode)
1005                ))
1006                .into());
1007            }
1008            // hint limitations for some other formats
1009            Format::Upsert => {
1010                notice_to_user(format!(
1011                    "Streaming queries on sources with `FORMAT {}` may have limitations. If your query isn't supported, consider using `CREATE TABLE` instead.",
1012                    format_encode.format
1013                ));
1014            }
1015            _ => {}
1016        }
1017    }
1018
1019    let sql_pk_names = bind_sql_pk_names(sql_columns_defs, bind_table_constraints(&constraints)?)?;
1020
1021    if with_properties.is_iceberg_connector() {
1022        if is_create_source
1023            && !with_properties.is_iceberg_update_source()
1024            && !sql_pk_names.is_empty()
1025        {
1026            return Err(ErrorCode::NotSupported(
1027                "PRIMARY KEY is not supported for Iceberg CREATE SOURCE in continuous ingestion mode."
1028                    .to_owned(),
1029                "Iceberg streaming ingestion only supports append-only sources. Remove the PRIMARY KEY clause."
1030                    .to_owned(),
1031            )
1032            .into());
1033        }
1034
1035        // FIXME: ideally we can support it, but current way of handling iceberg additional columns
1036        // are problematic. They are treated as normal user columns, so they will be lost if we
1037        // allow user to specify columns. See `extract_iceberg_columns`.
1038        if !sql_columns_defs.is_empty() {
1039            return Err(RwError::from(InvalidInputSyntax(
1040                r#"Schema is automatically inferred for iceberg source and should not be specified
1041
1042HINT: use `CREATE SOURCE <name> WITH (...)` instead of `CREATE SOURCE <name> (<columns>) WITH (...)`."#.to_owned(),
1043            )));
1044        }
1045    }
1046
1047    // Same for ADBC Snowflake connector - schema is automatically inferred
1048    if with_properties.is_batch_connector()
1049        && with_properties
1050            .get(UPSTREAM_SOURCE_KEY)
1051            .is_some_and(|s| s.eq_ignore_ascii_case(ADBC_SNOWFLAKE_CONNECTOR))
1052        && !sql_columns_defs.is_empty()
1053    {
1054        return Err(RwError::from(InvalidInputSyntax(
1055            r#"Schema is automatically inferred for ADBC Snowflake source and should not be specified
1056
1057HINT: use `CREATE TABLE <name> WITH (...)` instead of `CREATE TABLE <name> (<columns>) WITH (...)`."#.to_owned(),
1058        )));
1059    }
1060    let columns_from_sql = bind_sql_columns(sql_columns_defs, false)?;
1061
1062    reject_variant_columns_for_unsupported_encoding(&format_encode, &columns_from_sql)?;
1063
1064    let mut columns = bind_all_columns(
1065        &format_encode,
1066        columns_from_resolve_source,
1067        columns_from_sql,
1068        sql_columns_defs,
1069        wildcard_idx,
1070        sql_column_strategy,
1071    )?;
1072
1073    // add additional columns before bind pk, because `format upsert` requires the key column
1074    handle_addition_columns(
1075        Some(&format_encode),
1076        &with_properties,
1077        include_column_options,
1078        &mut columns,
1079        false,
1080    )?;
1081
1082    if columns.is_empty() {
1083        return Err(RwError::from(ProtocolError(
1084            "Schema definition is required, either from SQL or schema registry.".to_owned(),
1085        )));
1086    }
1087
1088    // compatible with the behavior that add a hidden column `_rw_kafka_timestamp` to each message from Kafka source
1089    if is_create_source {
1090        // must behind `handle_addition_columns`
1091        check_and_add_timestamp_column(&with_properties, &mut columns);
1092
1093        // For shared sources, we will include partition and offset cols in the SourceExecutor's *output*, to be used by the SourceBackfillExecutor.
1094        // For shared CDC source, the schema is different. See ColumnCatalog::debezium_cdc_source_cols(), CDC_BACKFILL_TABLE_ADDITIONAL_COLUMNS
1095        if create_source_type == CreateSourceType::SharedNonCdc {
1096            let (columns_exist, additional_columns) = source_add_partition_offset_cols(
1097                &columns,
1098                &with_properties.get_connector().unwrap(),
1099                true, // col_id filled below at col_id_gen.generate
1100            );
1101            for (existed, c) in columns_exist.into_iter().zip_eq_fast(additional_columns) {
1102                if !existed {
1103                    columns.push(ColumnCatalog::hidden(c));
1104                }
1105            }
1106        }
1107    }
1108
1109    // resolve privatelink connection for Kafka
1110    let mut with_properties = with_properties;
1111    resolve_privatelink_in_with_option(&mut with_properties)?;
1112
1113    // check the system parameter `enforce_secret`
1114    if session
1115        .env()
1116        .system_params_manager()
1117        .get_params()
1118        .load()
1119        .enforce_secret()
1120        && Feature::SecretManagement.check_available().is_ok()
1121    {
1122        // check enforce using secret for some props on cloud
1123        ConnectorProperties::enforce_secret_source(&with_properties)?;
1124    }
1125
1126    let (with_properties, connection_type, connector_conn_ref) =
1127        resolve_connection_ref_and_secret_ref(
1128            with_properties,
1129            session,
1130            Some(TelemetryDatabaseObject::Source),
1131        )?;
1132    ensure_connection_type_allowed(connection_type, &SOURCE_ALLOWED_CONNECTION_CONNECTOR)?;
1133
1134    // if not using connection, we don't need to check connector match connection type
1135    if !matches!(connection_type, PbConnectionType::Unspecified) {
1136        let Some(connector) = with_properties.get_connector() else {
1137            return Err(RwError::from(ProtocolError(format!(
1138                "missing field '{}' in WITH clause",
1139                UPSTREAM_SOURCE_KEY
1140            ))));
1141        };
1142        check_connector_match_connection_type(connector.as_str(), &connection_type)?;
1143    }
1144
1145    let pk_names = bind_source_pk(
1146        &format_encode,
1147        &source_info,
1148        &mut columns,
1149        sql_pk_names,
1150        &with_properties,
1151    )
1152    .await?;
1153
1154    // User may specify a generated or additional column with the same name as one from the external schema.
1155    // Ensure duplicated column names are handled here.
1156    if let Some(duplicated_name) = columns.iter().map(|c| c.name()).duplicates().next() {
1157        return Err(ErrorCode::InvalidInputSyntax(format!(
1158            "column \"{}\" specified more than once",
1159            duplicated_name
1160        ))
1161        .into());
1162    }
1163
1164    // XXX: why do we use col_id_gen here? It doesn't seem to be very necessary.
1165    for c in &mut columns {
1166        let original_data_type = c.data_type().clone();
1167        col_id_gen.generate(c)?;
1168        // TODO: Now we restore the data type for `CREATE SOURCE`, so that keep the nested field id unset.
1169        //       This behavior is inconsistent with `CREATE TABLE`, and should be fixed once we refactor
1170        //       `ALTER SOURCE` to also use `ColumnIdGenerator` in the future.
1171        if is_create_source {
1172            c.column_desc.data_type = original_data_type;
1173        }
1174    }
1175    debug_assert_column_ids_distinct(&columns);
1176
1177    let (mut columns, pk_col_ids, row_id_index) =
1178        bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
1179
1180    let watermark_descs =
1181        bind_source_watermark(session, source_name.clone(), source_watermarks, &columns)?;
1182    // TODO(yuhao): allow multiple watermark on source.
1183    assert!(watermark_descs.len() <= 1);
1184    if is_create_source && watermark_descs.iter().any(|d| d.with_ttl) {
1185        return Err(ErrorCode::NotSupported(
1186            "WITH TTL is not supported in WATERMARK clause for CREATE SOURCE.".to_owned(),
1187            "Use `CREATE TABLE ... WATERMARK ... WITH TTL` instead.".to_owned(),
1188        )
1189        .into());
1190    }
1191
1192    let append_only = row_id_index.is_some();
1193    if is_create_source && !append_only && !watermark_descs.is_empty() {
1194        return Err(ErrorCode::NotSupported(
1195            "Defining watermarks on source requires the source connector to be append only."
1196                .to_owned(),
1197            "Use the key words `FORMAT PLAIN`".to_owned(),
1198        )
1199        .into());
1200    }
1201
1202    bind_sql_column_constraints(
1203        session,
1204        source_name.clone(),
1205        &mut columns,
1206        // TODO(st1page): pass the ref
1207        sql_columns_defs,
1208        &pk_col_ids,
1209    )?;
1210    check_format_encode(&with_properties, row_id_index, &columns)?;
1211
1212    let definition = handler_args.normalized_sql.clone();
1213
1214    let associated_table_id = if is_create_source {
1215        None
1216    } else {
1217        Some(TableId::placeholder())
1218    };
1219    let source = SourceCatalog {
1220        id: SourceId::placeholder(),
1221        name: source_name,
1222        schema_id,
1223        database_id,
1224        columns,
1225        pk_col_ids,
1226        append_only,
1227        owner: session.user_id(),
1228        info: source_info,
1229        row_id_index,
1230        with_properties,
1231        watermark_descs,
1232        associated_table_id,
1233        definition,
1234        connection_id: connector_conn_ref,
1235        created_at_epoch: None,
1236        initialized_at_epoch: None,
1237        version: INITIAL_SOURCE_VERSION_ID,
1238        created_at_cluster_version: None,
1239        initialized_at_cluster_version: None,
1240        rate_limit: source_rate_limit,
1241        ingest_mode: Some(ingest_mode),
1242    };
1243    source.validate_iceberg_update_source()?;
1244    Ok(source)
1245}
1246
1247pub async fn handle_create_source(
1248    mut handler_args: HandlerArgs,
1249    stmt: CreateSourceStatement,
1250) -> Result<RwPgResponse> {
1251    let session = handler_args.session.clone();
1252
1253    if let Either::Right(resp) = session.check_relation_name_duplicated(
1254        stmt.source_name.clone(),
1255        StatementType::CREATE_SOURCE,
1256        stmt.if_not_exists,
1257    )? {
1258        return Ok(resp);
1259    }
1260
1261    if stmt
1262        .source_name
1263        .base_name()
1264        .starts_with(ICEBERG_SOURCE_PREFIX)
1265    {
1266        return Err(RwError::from(InvalidInputSyntax(format!(
1267            "Source name cannot start with reserved prefix '{}'",
1268            ICEBERG_SOURCE_PREFIX
1269        ))));
1270    }
1271
1272    if stmt.cdc_table_info.is_some() {
1273        return handle_create_cdc_table_source(handler_args, stmt).await;
1274    }
1275
1276    let overwrite_options = OverwriteOptions::new(&mut handler_args)?;
1277
1278    if handler_args.with_options.is_empty() {
1279        return Err(RwError::from(InvalidInputSyntax(
1280            "missing WITH clause".to_owned(),
1281        )));
1282    }
1283
1284    if overwrite_options.source_rate_limit == Some(0)
1285        && dispatch_cdc_source_type!(
1286            handler_args.with_options.cdc_source_type(),
1287            T,
1288            T::NEEDS_NON_EMPTY_INITIAL_OFFSET,
1289            false
1290        )
1291    {
1292        let connector = handler_args.with_options.get_connector().unwrap();
1293        return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
1294            "`source_rate_limit` cannot be 0 when creating a `{connector}` source because source creation must wait for the initial CDC offset."
1295        ))));
1296    }
1297
1298    let format_encode = stmt.format_encode.into_v2_with_warning();
1299    let (with_properties, ingest_mode) = bind_connector_props(&handler_args, &format_encode, true)?;
1300    if let Some(connector) = with_properties.get_connector() {
1301        ensure_local_fs_connector_allowed(&session, &connector)?;
1302    }
1303
1304    // Check the CREATE privilege before `bind_columns_from_source`, which may fetch from
1305    // user-supplied URLs (schema registry, schema file).
1306    let (schema_name, _) =
1307        Binder::resolve_schema_qualified_name(&session.database(), &stmt.source_name)?;
1308    session.get_database_and_schema_id_for_create(schema_name)?;
1309
1310    let create_source_type = CreateSourceType::for_newly_created(&session, &*with_properties);
1311    let (columns_from_resolve_source, source_info) = bind_columns_from_source(
1312        &session,
1313        &format_encode,
1314        Either::Left(&with_properties),
1315        create_source_type,
1316    )
1317    .await?;
1318    let mut col_id_gen = ColumnIdGenerator::new_initial();
1319
1320    if stmt.columns.iter().any(|col| {
1321        col.options
1322            .iter()
1323            .any(|def| matches!(def.option, ColumnOption::NotNull))
1324    }) {
1325        return Err(RwError::from(InvalidInputSyntax(
1326            "NOT NULL constraint is not supported in source schema".to_owned(),
1327        )));
1328    }
1329
1330    let source_catalog = bind_create_source_or_table_with_connector(
1331        handler_args.clone(),
1332        stmt.source_name,
1333        format_encode,
1334        with_properties,
1335        &stmt.columns,
1336        stmt.constraints,
1337        stmt.wildcard_idx,
1338        stmt.source_watermarks,
1339        columns_from_resolve_source,
1340        source_info,
1341        stmt.include_column_options,
1342        &mut col_id_gen,
1343        create_source_type,
1344        overwrite_options.source_rate_limit,
1345        SqlColumnStrategy::FollowChecked,
1346        ingest_mode,
1347    )
1348    .await?;
1349
1350    // If it is a temporary source, put it into SessionImpl.
1351    if stmt.temporary {
1352        if session.get_temporary_source(&source_catalog.name).is_some() {
1353            return Err(CatalogError::duplicated("source", source_catalog.name.clone()).into());
1354        }
1355        session.create_temporary_source(source_catalog);
1356        return Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE));
1357    }
1358
1359    let source = source_catalog.to_prost();
1360
1361    let catalog_writer = session.catalog_writer()?;
1362
1363    if create_source_type.is_shared() {
1364        let graph = generate_stream_graph_for_source(handler_args, source_catalog)?;
1365        catalog_writer
1366            .create_source(source, Some(graph), stmt.if_not_exists)
1367            .await?;
1368    } else {
1369        // For other sources we don't create a streaming job
1370        catalog_writer
1371            .create_source(source, None, stmt.if_not_exists)
1372            .await?;
1373    }
1374
1375    Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE))
1376}
1377
1378async fn handle_create_cdc_table_source(
1379    handler_args: HandlerArgs,
1380    stmt: CreateSourceStatement,
1381) -> Result<RwPgResponse> {
1382    let session = handler_args.session.clone();
1383    let cdc_table_info = stmt.cdc_table_info.as_ref().expect("checked by caller");
1384
1385    if stmt.temporary {
1386        return Err(ErrorCode::NotSupported(
1387            "temporary CDC table sources are not supported".to_owned(),
1388            "Remove the TEMPORARY clause".to_owned(),
1389        )
1390        .into());
1391    }
1392
1393    if handler_args.with_options.len() != 1
1394        || !handler_args
1395            .with_options
1396            .value_eq_ignore_case(CDC_BACKFILL_ENABLE_KEY, "false")
1397        || !handler_args.with_options.secret_ref().is_empty()
1398        || !handler_args.with_options.connection_ref().is_empty()
1399    {
1400        return Err(ErrorCode::InvalidInputSyntax(
1401            "CDC table sources currently require exactly `WITH (snapshot = 'false')`".to_owned(),
1402        )
1403        .into());
1404    }
1405
1406    if !stmt.source_watermarks.is_empty() {
1407        return Err(ErrorCode::NotSupported(
1408            "watermarks on CDC table sources are not supported".to_owned(),
1409            "Remove the WATERMARK clause".to_owned(),
1410        )
1411        .into());
1412    }
1413    if stmt.columns.iter().any(|column| column.is_generated()) {
1414        return Err(ErrorCode::NotSupported(
1415            "generated columns on CDC table sources are not supported".to_owned(),
1416            "Define generated expressions in the downstream streaming query".to_owned(),
1417        )
1418        .into());
1419    }
1420    for column in &stmt.columns {
1421        for option in &column.options {
1422            if matches!(
1423                option.option,
1424                ColumnOption::DefaultValue(_) | ColumnOption::DefaultValueInternal { .. }
1425            ) {
1426                return Err(ErrorCode::NotSupported(
1427                    "default values on CDC table sources are not supported".to_owned(),
1428                    "Remove the default value expression".to_owned(),
1429                )
1430                .into());
1431            }
1432        }
1433    }
1434
1435    sanity_check_for_table_on_cdc_source(
1436        false,
1437        &stmt.columns,
1438        &stmt.wildcard_idx,
1439        &stmt.constraints,
1440        &stmt.source_watermarks,
1441    )?;
1442    not_null_check_for_cdc_table(&stmt.wildcard_idx, &stmt.columns)?;
1443
1444    let db_name = session.database();
1445    let user_name = session.user_name();
1446    let search_path = session.config().search_path();
1447    let (schema_name, source_name) =
1448        Binder::resolve_schema_qualified_name(&db_name, &stmt.source_name)?;
1449    let (database_id, schema_id) =
1450        session.get_database_and_schema_id_for_create(schema_name.clone())?;
1451
1452    let (upstream_schema, upstream_source_name) =
1453        Binder::resolve_schema_qualified_name(&db_name, &cdc_table_info.source_name)?;
1454    let upstream_source = {
1455        let catalog_reader = session.env().catalog_reader().read_guard();
1456        let schema_path = SchemaPath::new(upstream_schema.as_deref(), &search_path, &user_name);
1457        let (source, _) = catalog_reader.get_source_by_name(
1458            &db_name,
1459            schema_path,
1460            upstream_source_name.as_str(),
1461        )?;
1462        source.clone()
1463    };
1464    check_cdc_source_select_privilege(&session, &upstream_source)?;
1465    if !upstream_source.info.is_shared()
1466        || !upstream_source.with_properties.is_shareable_cdc_connector()
1467        || upstream_source.info.external_table.is_some()
1468    {
1469        return Err(ErrorCode::InvalidInputSyntax(format!(
1470            "source `{}` is not a shared CDC source",
1471            cdc_table_info.source_name
1472        ))
1473        .into());
1474    }
1475
1476    let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1477        &upstream_source.with_properties,
1478        cdc_table_info.external_table_name.clone(),
1479    )?;
1480    let (mut columns, pk_names, pk_comparisons) = match stmt.wildcard_idx {
1481        Some(_) => {
1482            bind_cdc_table_schema_externally(cdc_with_options.clone(), &HashSet::new()).await?
1483        }
1484        None => {
1485            let (columns, pk_names) =
1486                bind_cdc_table_schema(&stmt.columns, &stmt.constraints, false)?;
1487            let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1488                cdc_with_options.clone(),
1489                &pk_names,
1490            ))
1491            .await?;
1492            (columns, pk_names, pk_comparisons)
1493        }
1494    };
1495    if pk_names.is_empty() {
1496        return Err(ErrorCode::NotSupported(
1497            "CDC table source without a primary key is not supported".to_owned(),
1498            "Define a primary key on the upstream table or in the source schema".to_owned(),
1499        )
1500        .into());
1501    }
1502    reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
1503
1504    handle_addition_columns(
1505        None,
1506        &cdc_with_options,
1507        stmt.include_column_options.clone(),
1508        &mut columns,
1509        true,
1510    )?;
1511    let mut col_id_gen = ColumnIdGenerator::new_initial();
1512    for column in &mut columns {
1513        col_id_gen.generate(column)?;
1514    }
1515    debug_assert_column_ids_distinct(&columns);
1516    let (columns, pk_col_ids, row_id_index) =
1517        bind_pk_and_row_id_on_relation(columns, pk_names, false)?;
1518    debug_assert!(row_id_index.is_none());
1519
1520    let (connect_properties, secret_refs) = cdc_with_options.into_parts();
1521    let id_to_index = columns
1522        .iter()
1523        .enumerate()
1524        .map(|(index, column)| (column.column_id(), index))
1525        .collect::<HashMap<_, _>>();
1526    let stream_key = pk_col_ids.iter().map(|id| id_to_index[id]).collect_vec();
1527    let pk = stream_key
1528        .iter()
1529        .map(|index| {
1530            risingwave_common::util::sort_util::ColumnOrder::new(
1531                *index,
1532                risingwave_common::util::sort_util::OrderType::ascending(),
1533            )
1534        })
1535        .collect();
1536    let cdc_table_desc = CdcTableDesc {
1537        table_id: TableId::placeholder(),
1538        source_id: upstream_source.id,
1539        external_table_name,
1540        pk,
1541        pk_comparisons,
1542        columns: columns
1543            .iter()
1544            .map(|column| column.column_desc.clone())
1545            .collect(),
1546        stream_key,
1547        // Meta validates this descriptor against the upstream table with these properties and
1548        // then clears them before persisting the catalog: consumers resolve them from
1549        // `source_id` when they are planned, so a persisted copy would only go stale after
1550        // `ALTER SOURCE`.
1551        connect_properties,
1552        secret_refs,
1553    };
1554
1555    let mut source_info = upstream_source.info.clone();
1556    source_info.cdc_source_job = false;
1557    source_info.is_distributed = false;
1558    source_info.external_table = Some(cdc_table_desc.to_protobuf());
1559
1560    // A catalog-only table source keeps just enough connector identity for catalog display and
1561    // source-kind checks. Connection details belong exclusively to the upstream shared source.
1562    let catalog_with_properties = WithOptionsSecResolved::new(
1563        BTreeMap::from([
1564            (
1565                UPSTREAM_SOURCE_KEY.to_owned(),
1566                upstream_source
1567                    .with_properties
1568                    .get_connector()
1569                    .expect("validated as a CDC source"),
1570            ),
1571            (CDC_BACKFILL_ENABLE_KEY.to_owned(), "false".to_owned()),
1572        ]),
1573        BTreeMap::new(),
1574    );
1575
1576    let source_catalog = SourceCatalog {
1577        id: SourceId::placeholder(),
1578        name: source_name,
1579        schema_id,
1580        database_id,
1581        columns,
1582        pk_col_ids,
1583        append_only: false,
1584        owner: session.user_id(),
1585        info: source_info,
1586        row_id_index: None,
1587        with_properties: catalog_with_properties,
1588        watermark_descs: vec![],
1589        associated_table_id: None,
1590        definition: handler_args.normalized_sql,
1591        connection_id: None,
1592        created_at_epoch: None,
1593        initialized_at_epoch: None,
1594        version: INITIAL_SOURCE_VERSION_ID,
1595        created_at_cluster_version: None,
1596        initialized_at_cluster_version: None,
1597        rate_limit: None,
1598        ingest_mode: upstream_source.ingest_mode,
1599    };
1600
1601    session
1602        .catalog_writer()?
1603        .create_source(source_catalog.to_prost(), None, stmt.if_not_exists)
1604        .await?;
1605    Ok(PgResponse::empty_result(StatementType::CREATE_SOURCE))
1606}
1607
1608pub(super) fn generate_stream_graph_for_source(
1609    handler_args: HandlerArgs,
1610    source_catalog: SourceCatalog,
1611) -> Result<PbStreamFragmentGraph> {
1612    let context = OptimizerContext::from_handler_args(handler_args)?;
1613    let source_node = LogicalSource::with_catalog(
1614        Rc::new(source_catalog),
1615        SourceNodeKind::CreateSharedSource,
1616        context.into(),
1617        None,
1618    )?;
1619
1620    let stream_plan = source_node.to_stream(&mut ToStreamContext::new_with_backfill_type(
1621        false,
1622        // Shared source planning does not create stream table scans, so this required context
1623        // value is only a placeholder and is not used for backfill selection.
1624        BackfillType::ArrangementBackfill,
1625    ))?;
1626    let graph = build_graph(stream_plan, Some(GraphJobType::Source))?;
1627    Ok(graph)
1628}
1629
1630#[cfg(all(test, not(madsim)))]
1631#[path = "create_source/iceberg_update_test.rs"]
1632mod iceberg_update_tests;
1633
1634#[cfg(test)]
1635pub mod tests {
1636    use std::collections::HashMap;
1637    use std::sync::Arc;
1638
1639    use risingwave_common::catalog::{
1640        DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, ROW_ID_COLUMN_NAME,
1641    };
1642    use risingwave_common::config::FrontendConfig;
1643    use risingwave_common::types::{DataType, StructType};
1644    use risingwave_connector::parser::additional_columns::get_kafka_header_item_datatype;
1645    use risingwave_pb::plan_common::EncodeType;
1646
1647    use crate::catalog::root_catalog::SchemaPath;
1648    use crate::catalog::source_catalog::SourceCatalog;
1649    use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
1650
1651    const GET_COLUMN_FROM_CATALOG: fn(&Arc<SourceCatalog>) -> HashMap<&str, DataType> =
1652        |catalog: &Arc<SourceCatalog>| -> HashMap<&str, DataType> {
1653            catalog
1654                .columns
1655                .iter()
1656                .map(|col| (col.name(), col.data_type().clone()))
1657                .collect::<HashMap<&str, DataType>>()
1658        };
1659
1660    #[tokio::test]
1661    async fn test_create_source_handler() {
1662        let proto_file = create_proto_file(PROTO_FILE_DATA);
1663        let sql = format!(
1664            r#"CREATE SOURCE t
1665    WITH (connector = 'kinesis')
1666    FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecord', schema.location = 'file://{}')"#,
1667            proto_file.path().to_str().unwrap()
1668        );
1669        let frontend = LocalFrontend::new(Default::default()).await;
1670        frontend.run_sql(sql).await.unwrap();
1671
1672        let session = frontend.session_ref();
1673        let catalog_reader = session.env().catalog_reader().read_guard();
1674        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1675
1676        // Check source exists.
1677        let (source, _) = catalog_reader
1678            .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
1679            .unwrap();
1680        assert_eq!(source.name, "t");
1681
1682        let columns = GET_COLUMN_FROM_CATALOG(source);
1683
1684        let city_type = StructType::new(vec![
1685            ("address", DataType::Varchar),
1686            ("zipcode", DataType::Varchar),
1687        ])
1688        // .with_ids([5, 6].map(ColumnId::new))
1689        .into();
1690        let expected_columns = maplit::hashmap! {
1691            ROW_ID_COLUMN_NAME => DataType::Serial,
1692            "id" => DataType::Int32,
1693            "zipcode" => DataType::Int64,
1694            "rate" => DataType::Float32,
1695            "country" => StructType::new(
1696                vec![("address", DataType::Varchar),("city", city_type),("zipcode", DataType::Varchar)],
1697            )
1698            // .with_ids([3, 4, 7].map(ColumnId::new))
1699            .into(),
1700        };
1701        assert_eq!(columns, expected_columns, "{columns:#?}");
1702    }
1703
1704    #[tokio::test]
1705    async fn test_create_mqtt_source_with_protobuf() {
1706        let proto_file = create_proto_file(PROTO_FILE_DATA);
1707        let sql = format!(
1708            r#"CREATE SOURCE t_mqtt
1709    WITH (
1710        connector = 'mqtt',
1711        url = 'mqtt://localhost:1883',
1712        topic = 'test_topic'
1713    )
1714    FORMAT PLAIN ENCODE PROTOBUF (
1715        message = '.test.TestRecord',
1716        schema.location = 'file://{}'
1717    )"#,
1718            proto_file.path().to_str().unwrap()
1719        );
1720        let frontend = LocalFrontend::new(Default::default()).await;
1721        frontend.run_sql(sql).await.unwrap();
1722
1723        let session = frontend.session_ref();
1724        let catalog_reader = session.env().catalog_reader().read_guard();
1725        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1726
1727        let (source, _) = catalog_reader
1728            .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t_mqtt")
1729            .unwrap();
1730
1731        assert_eq!(source.name, "t_mqtt");
1732        assert_eq!(source.info.row_encode, EncodeType::Protobuf as i32);
1733    }
1734
1735    #[tokio::test]
1736    async fn test_create_posix_fs_source_requires_frontend_config() {
1737        let frontend = LocalFrontend::with_frontend_config(
1738            Default::default(),
1739            FrontendConfig {
1740                unsafe_enable_local_fs_connector: false,
1741                ..Default::default()
1742            },
1743        )
1744        .await;
1745        let err = frontend
1746            .run_sql(
1747                r#"CREATE SOURCE local_files (
1748                    line VARCHAR
1749                ) WITH (
1750                    connector = 'posix_fs',
1751                    posix_fs.root = '/tmp',
1752                    match_pattern = '*.csv'
1753                ) FORMAT PLAIN ENCODE CSV (without_header = 'true')"#
1754                    .to_owned(),
1755            )
1756            .await
1757            .unwrap_err();
1758
1759        assert!(
1760            err.to_string()
1761                .contains("frontend.unsafe_enable_local_fs_connector = true"),
1762            "{err:?}"
1763        );
1764    }
1765
1766    #[tokio::test]
1767    async fn test_duplicate_props_options() {
1768        let proto_file = create_proto_file(PROTO_FILE_DATA);
1769        let sql = format!(
1770            r#"CREATE SOURCE t
1771    WITH (
1772        connector = 'kinesis',
1773        aws.region='user_test_topic',
1774        endpoint='172.10.1.1:9090,172.10.1.2:9090',
1775        aws.credentials.access_key_id = 'your_access_key_1',
1776        aws.credentials.secret_access_key = 'your_secret_key_1'
1777    )
1778    FORMAT PLAIN ENCODE PROTOBUF (
1779        message = '.test.TestRecord',
1780        aws.credentials.access_key_id = 'your_access_key_2',
1781        aws.credentials.secret_access_key = 'your_secret_key_2',
1782        schema.location = 'file://{}',
1783    )"#,
1784            proto_file.path().to_str().unwrap()
1785        );
1786        let frontend = LocalFrontend::new(Default::default()).await;
1787        frontend.run_sql(sql).await.unwrap();
1788
1789        let session = frontend.session_ref();
1790        let catalog_reader = session.env().catalog_reader().read_guard();
1791        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1792
1793        // Check source exists.
1794        let (source, _) = catalog_reader
1795            .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
1796            .unwrap();
1797        assert_eq!(source.name, "t");
1798
1799        // AwsAuth params exist in options.
1800        assert_eq!(
1801            source
1802                .info
1803                .format_encode_options
1804                .get("aws.credentials.access_key_id")
1805                .unwrap(),
1806            "your_access_key_2"
1807        );
1808        assert_eq!(
1809            source
1810                .info
1811                .format_encode_options
1812                .get("aws.credentials.secret_access_key")
1813                .unwrap(),
1814            "your_secret_key_2"
1815        );
1816
1817        // AwsAuth params exist in props.
1818        assert_eq!(
1819            source
1820                .with_properties
1821                .get("aws.credentials.access_key_id")
1822                .unwrap(),
1823            "your_access_key_1"
1824        );
1825        assert_eq!(
1826            source
1827                .with_properties
1828                .get("aws.credentials.secret_access_key")
1829                .unwrap(),
1830            "your_secret_key_1"
1831        );
1832
1833        // Options are not merged into props.
1834        assert!(!source.with_properties.contains_key("schema.location"));
1835    }
1836
1837    #[tokio::test]
1838    async fn test_multi_table_cdc_create_source_handler() {
1839        let sql =
1840            "CREATE SOURCE t2 WITH (connector = 'mysql-cdc') FORMAT PLAIN ENCODE JSON".to_owned();
1841        let frontend = LocalFrontend::new(Default::default()).await;
1842        let session = frontend.session_ref();
1843
1844        frontend
1845            .run_sql_with_session(session.clone(), sql)
1846            .await
1847            .unwrap();
1848        let catalog_reader = session.env().catalog_reader().read_guard();
1849        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
1850
1851        // Check source exists.
1852        let (source, _) = catalog_reader
1853            .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t2")
1854            .unwrap();
1855        assert_eq!(source.name, "t2");
1856
1857        let columns = source
1858            .columns
1859            .iter()
1860            .map(|col| (col.name(), col.data_type().clone()))
1861            .collect::<Vec<(&str, DataType)>>();
1862
1863        expect_test::expect![[r#"
1864            [
1865                (
1866                    "payload",
1867                    Jsonb,
1868                ),
1869                (
1870                    "_rw_offset",
1871                    Varchar,
1872                ),
1873                (
1874                    "_rw_table_name",
1875                    Varchar,
1876                ),
1877                (
1878                    "_row_id",
1879                    Serial,
1880                ),
1881            ]
1882        "#]]
1883        .assert_debug_eq(&columns);
1884    }
1885
1886    #[tokio::test]
1887    async fn test_reject_zero_source_rate_limit_when_cdc_must_wait_for_offset() {
1888        let frontend = LocalFrontend::new(Default::default()).await;
1889        let session = frontend.session_ref();
1890
1891        for (source_name, connector) in [
1892            ("mysql_explicit_zero", "mysql-cdc"),
1893            ("sqlserver_explicit_zero", "sqlserver-cdc"),
1894        ] {
1895            let err = frontend
1896                .run_sql_with_session(
1897                    session.clone(),
1898                    format!(
1899                        "CREATE SOURCE {source_name} WITH (connector = '{connector}', source_rate_limit = '0') FORMAT PLAIN ENCODE JSON"
1900                    ),
1901                )
1902                .await
1903                .unwrap_err();
1904            let message = err.to_string();
1905            assert!(
1906                message.contains("source creation must wait for the initial CDC offset"),
1907                "{message}"
1908            );
1909        }
1910
1911        frontend
1912            .run_sql_with_session(session.clone(), "SET source_rate_limit TO 0;")
1913            .await
1914            .unwrap();
1915
1916        for (source_name, connector) in [
1917            ("mysql_session_zero", "mysql-cdc"),
1918            ("sqlserver_session_zero", "sqlserver-cdc"),
1919        ] {
1920            let err = frontend
1921                .run_sql_with_session(
1922                    session.clone(),
1923                    format!(
1924                        "CREATE SOURCE {source_name} WITH (connector = '{connector}') FORMAT PLAIN ENCODE JSON"
1925                    ),
1926                )
1927                .await
1928                .unwrap_err();
1929            assert!(
1930                err.to_string()
1931                    .contains("source creation must wait for the initial CDC offset"),
1932                "{err}"
1933            );
1934        }
1935
1936        frontend
1937            .run_sql_with_session(
1938                session.clone(),
1939                "CREATE SOURCE mysql_positive_override WITH (connector = 'mysql-cdc', source_rate_limit = '1') FORMAT PLAIN ENCODE JSON",
1940            )
1941            .await
1942            .unwrap();
1943
1944        frontend
1945            .run_sql_with_session(
1946                session,
1947                "CREATE SOURCE postgres_zero WITH (connector = 'postgres-cdc') FORMAT PLAIN ENCODE JSON",
1948            )
1949            .await
1950            .unwrap();
1951    }
1952
1953    #[tokio::test]
1954    async fn test_source_addition_columns() {
1955        // test derive include column for format plain
1956        let sql =
1957            "CREATE SOURCE s (v1 int) include key as _rw_kafka_key with (connector = 'kafka') format plain encode json".to_owned();
1958        let frontend = LocalFrontend::new(Default::default()).await;
1959        frontend.run_sql(sql).await.unwrap();
1960        let session = frontend.session_ref();
1961        let catalog_reader = session.env().catalog_reader().read_guard();
1962        let (source, _) = catalog_reader
1963            .get_source_by_name(
1964                DEFAULT_DATABASE_NAME,
1965                SchemaPath::Name(DEFAULT_SCHEMA_NAME),
1966                "s",
1967            )
1968            .unwrap();
1969        assert_eq!(source.name, "s");
1970
1971        let columns = source
1972            .columns
1973            .iter()
1974            .map(|col| (col.name(), col.data_type().clone()))
1975            .collect::<Vec<(&str, DataType)>>();
1976
1977        expect_test::expect![[r#"
1978            [
1979                (
1980                    "v1",
1981                    Int32,
1982                ),
1983                (
1984                    "_rw_kafka_key",
1985                    Bytea,
1986                ),
1987                (
1988                    "_rw_kafka_timestamp",
1989                    Timestamptz,
1990                ),
1991                (
1992                    "_rw_kafka_partition",
1993                    Varchar,
1994                ),
1995                (
1996                    "_rw_kafka_offset",
1997                    Varchar,
1998                ),
1999                (
2000                    "_row_id",
2001                    Serial,
2002                ),
2003            ]
2004        "#]]
2005        .assert_debug_eq(&columns);
2006        drop(catalog_reader);
2007
2008        let sql =
2009            "CREATE SOURCE s_pulsar (v1 int) include header 'tenant' as pulsar_header with (connector = 'pulsar') format plain encode json".to_owned();
2010        frontend.run_sql(sql).await.unwrap();
2011        let catalog_reader = session.env().catalog_reader().read_guard();
2012        let (source, _) = catalog_reader
2013            .get_source_by_name(
2014                DEFAULT_DATABASE_NAME,
2015                SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2016                "s_pulsar",
2017            )
2018            .unwrap();
2019        assert_eq!(source.name, "s_pulsar");
2020
2021        let columns = source
2022            .columns
2023            .iter()
2024            .map(|col| (col.name(), col.data_type().clone()))
2025            .collect::<Vec<(&str, DataType)>>();
2026
2027        expect_test::expect![[r#"
2028            [
2029                (
2030                    "v1",
2031                    Int32,
2032                ),
2033                (
2034                    "pulsar_header",
2035                    Bytea,
2036                ),
2037                (
2038                    "_row_id",
2039                    Serial,
2040                ),
2041            ]
2042        "#]]
2043        .assert_debug_eq(&columns);
2044        drop(catalog_reader);
2045
2046        let sql =
2047            "CREATE SOURCE s_nats (v1 int) include header 'trace-id' as nats_header with (connector = 'nats') format plain encode json".to_owned();
2048        frontend.run_sql(sql).await.unwrap();
2049        let catalog_reader = session.env().catalog_reader().read_guard();
2050        let (source, _) = catalog_reader
2051            .get_source_by_name(
2052                DEFAULT_DATABASE_NAME,
2053                SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2054                "s_nats",
2055            )
2056            .unwrap();
2057        assert_eq!(source.name, "s_nats");
2058
2059        let columns = source
2060            .columns
2061            .iter()
2062            .map(|col| (col.name(), col.data_type().clone()))
2063            .collect::<Vec<(&str, DataType)>>();
2064
2065        expect_test::expect![[r#"
2066            [
2067                (
2068                    "v1",
2069                    Int32,
2070                ),
2071                (
2072                    "nats_header",
2073                    Bytea,
2074                ),
2075                (
2076                    "_row_id",
2077                    Serial,
2078                ),
2079            ]
2080        "#]]
2081        .assert_debug_eq(&columns);
2082        drop(catalog_reader);
2083
2084        // Bare `INCLUDE header AS ...` -- list-of-struct dispatch through the NATS DDL path.
2085        let sql =
2086            "CREATE SOURCE s_nats_all (v1 int) include header as nats_all_headers with (connector = 'nats') format plain encode json".to_owned();
2087        frontend.run_sql(sql).await.unwrap();
2088        let catalog_reader = session.env().catalog_reader().read_guard();
2089        let (source, _) = catalog_reader
2090            .get_source_by_name(
2091                DEFAULT_DATABASE_NAME,
2092                SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2093                "s_nats_all",
2094            )
2095            .unwrap();
2096        assert_eq!(source.name, "s_nats_all");
2097        let all_headers_col = source
2098            .columns
2099            .iter()
2100            .find(|c| c.name() == "nats_all_headers")
2101            .expect("nats_all_headers column present");
2102        assert_eq!(
2103            all_headers_col.data_type(),
2104            &DataType::list(get_kafka_header_item_datatype()),
2105        );
2106        drop(catalog_reader);
2107
2108        let sql =
2109            "CREATE SOURCE s3 (v1 int) include timestamp 'header1' as header_col with (connector = 'kafka') format plain encode json".to_owned();
2110        match frontend.run_sql(sql).await {
2111            Err(e) => {
2112                assert_eq!(
2113                    e.to_string(),
2114                    "Protocol error: Only header column can have inner field, but got \"timestamp\""
2115                )
2116            }
2117            _ => unreachable!(),
2118        }
2119    }
2120
2121    /// The CREATE privilege must be checked before the schema is fetched from a user-supplied
2122    /// URL (schema registry or schema file). Otherwise a user without CREATE on the schema can
2123    /// still make the frontend send the request, and only then get "permission denied".
2124    #[tokio::test]
2125    async fn test_create_privilege_checked_before_schema_fetch() {
2126        let frontend = LocalFrontend::new(Default::default()).await;
2127        frontend
2128            .run_sql("CREATE USER no_create_user")
2129            .await
2130            .unwrap();
2131        let user_id = frontend
2132            .session_ref()
2133            .env()
2134            .user_info_reader()
2135            .read_guard()
2136            .get_user_by_name("no_create_user")
2137            .unwrap()
2138            .id;
2139        let user_session = frontend.session_user_ref(
2140            DEFAULT_DATABASE_NAME.to_owned(),
2141            "no_create_user".to_owned(),
2142            user_id,
2143        );
2144
2145        // Nothing listens on this port, so if the request is sent at all it fails with a
2146        // connection error instead of "permission denied".
2147        let with_and_format = "WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:1', topic = 't') \
2148             FORMAT PLAIN ENCODE AVRO (schema.registry = 'http://127.0.0.1:1')";
2149        let create_source = format!("CREATE SOURCE s {with_and_format}");
2150        let create_table = format!("CREATE TABLE t {with_and_format}");
2151
2152        for sql in [create_source, create_table] {
2153            let err = frontend
2154                .run_sql_with_session(user_session.clone(), sql)
2155                .await
2156                .unwrap_err();
2157            assert!(
2158                err.to_string()
2159                    .contains("permission denied for schema \"public\": \"CREATE\""),
2160                "{err:?}"
2161            );
2162        }
2163    }
2164}