Skip to main content

risingwave_frontend/handler/
create_table.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, HashMap, HashSet};
16use std::rc::Rc;
17use std::sync::Arc;
18
19use anyhow::Context;
20use clap::ValueEnum;
21use either::Either;
22use fixedbitset::FixedBitSet;
23use iceberg::spec::FormatVersion;
24use itertools::Itertools;
25use percent_encoding::percent_decode_str;
26use pgwire::pg_response::{PgResponse, StatementType};
27use prost::Message as _;
28use risingwave_common::acl::AclMode;
29use risingwave_common::catalog::{
30    CdcKeyComparison, CdcTableDesc, ColumnCatalog, ColumnDesc, ConflictBehavior,
31    DEFAULT_SCHEMA_NAME, Engine, ICEBERG_SINK_PREFIX, ICEBERG_SOURCE_PREFIX,
32    RISINGWAVE_ICEBERG_ROW_ID, ROW_ID_COLUMN_NAME, TableId,
33};
34use risingwave_common::config::MetaBackend;
35use risingwave_common::global_jvm::Jvm;
36use risingwave_common::session_config::sink_decouple::SinkDecouple;
37use risingwave_common::types::DataType;
38use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
39use risingwave_common::util::value_encoding::DatumToProtoExt;
40use risingwave_common::{bail, bail_not_implemented};
41use risingwave_connector::source::cdc::external::ExternalCdcTableType;
42use risingwave_connector::source::cdc::{
43    CDC_COLUMN_EXCLUDE_LIST, CdcSourceExternalTableIdentifier, build_cdc_table_id,
44    parse_cdc_column_exclude_list,
45};
46use risingwave_connector::{
47    AUTO_SCHEMA_CHANGE_KEY, WithOptionsSecResolved, WithPropertiesExt, dispatch_cdc_source_type,
48};
49use risingwave_pb::catalog::connection::Info as ConnectionInfo;
50use risingwave_pb::catalog::connection_params::ConnectionType;
51use risingwave_pb::catalog::{PbSource, PbWebhookSourceInfo, WatermarkDesc};
52use risingwave_pb::ddl_service::{PbTableJobType, TableJobType};
53use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
54use risingwave_pb::plan_common::{
55    AdditionalColumn, ColumnDescVersion, DefaultColumnDesc, GeneratedColumnDesc,
56};
57use risingwave_pb::secret::PbSecretRef;
58use risingwave_pb::secret::secret_ref::PbRefAsType;
59use risingwave_pb::stream_plan::StreamFragmentGraph;
60use risingwave_sqlparser::ast::{
61    CdcTableInfo, ColumnDef, ColumnOption, CompatibleFormatEncode, ConnectionRefValue, CreateSink,
62    CreateSinkStatement, CreateSourceStatement, DataType as AstDataType, ExplainOptions, Format,
63    FormatEncodeOptions, Ident, ObjectName, OnConflict, SecretRefAsType, SourceWatermark,
64    Statement, TableConstraint, WebhookSourceInfo, WithProperties,
65};
66use risingwave_sqlparser::parser::IncludeOption;
67use thiserror_ext::AsReport;
68
69use super::RwPgResponse;
70use super::create_source::{CreateSourceType, SqlColumnStrategy, bind_columns_from_source};
71use crate::binder::{Clause, SecureCompareContext, WEBHOOK_PAYLOAD_FIELD_NAME, bind_data_type};
72use crate::catalog::root_catalog::SchemaPath;
73use crate::catalog::source_catalog::SourceCatalog;
74use crate::catalog::table_catalog::TableVersion;
75use crate::catalog::{ColumnId, DatabaseId, SchemaId, SourceId, check_column_name_not_reserved};
76use crate::error::{ErrorCode, Result, RwError, bail_bind_error};
77use crate::expr::{Expr, ExprImpl, ExprRewriter};
78use crate::handler::HandlerArgs;
79use crate::handler::cdc::{
80    bind_cdc_pk_comparisons_externally, bind_cdc_table_schema, bind_cdc_table_schema_externally,
81    derive_connect_properties_for_cdc_table, not_null_check_for_cdc_table,
82    reject_pk_filtered_by_debezium_column_filter, sanity_check_for_table_on_cdc_source,
83};
84use crate::handler::create_source::{
85    bind_connector_props, bind_create_source_or_table_with_connector, bind_source_watermark,
86    handle_addition_columns,
87};
88use crate::handler::privilege::ObjectCheckItem;
89use crate::handler::util::{
90    LongRunningNotificationAction, SourceSchemaCompatExt, execute_with_long_running_notification,
91};
92use crate::optimizer::plan_node::generic::{SourceNodeKind, build_cdc_scan_options_with_options};
93use crate::optimizer::plan_node::{
94    LogicalCdcScan, LogicalPlanRef, LogicalSource, StreamPlanRef as PlanRef,
95    ensure_sync_log_store_fragment_root,
96};
97use crate::optimizer::property::{Order, RequiredDist};
98use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRoot};
99use crate::session::SessionImpl;
100use crate::session::current::notice_to_user;
101use crate::stream_fragmenter::{GraphJobType, build_graph};
102use crate::utils::OverwriteOptions;
103use crate::{Binder, Explain, TableCatalog, WithOptions};
104
105mod col_id_gen;
106pub use col_id_gen::*;
107use risingwave_connector::sink::SinkParam;
108use risingwave_connector::sink::iceberg::{
109    ENABLE_COMPACTION, ENABLE_MANIFEST_REWRITE, IcebergConfig, IcebergSink,
110    is_iceberg_engine_option, parse_partition_by_exprs, validate_order_key_columns,
111};
112use risingwave_pb::ddl_service::create_iceberg_table_request::{PbSinkJobInfo, PbTableJobInfo};
113
114use crate::handler::create_sink::{SinkPlanContext, gen_sink_plan};
115
116fn ensure_column_options_supported(c: &ColumnDef) -> Result<()> {
117    for option_def in &c.options {
118        match option_def.option {
119            ColumnOption::GeneratedColumns(_) => {}
120            ColumnOption::DefaultValue(_) => {}
121            ColumnOption::DefaultValueInternal { .. } => {}
122            ColumnOption::Unique { is_primary: true } => {}
123            ColumnOption::Null => {}
124            ColumnOption::NotNull => {}
125            _ => bail_not_implemented!("column constraints \"{}\"", option_def),
126        }
127    }
128    Ok(())
129}
130
131/// Binds the column schemas declared in CREATE statement into `ColumnDesc`.
132/// If a column is marked as `primary key`, its `ColumnId` is also returned.
133/// This primary key is not combined with table constraints yet.
134pub fn bind_sql_columns(
135    column_defs: &[ColumnDef],
136    is_for_drop_table_connector: bool,
137) -> Result<Vec<ColumnCatalog>> {
138    let mut columns = Vec::with_capacity(column_defs.len());
139
140    for column in column_defs {
141        ensure_column_options_supported(column)?;
142        // Destruct to make sure all fields are properly handled rather than ignored.
143        // Do NOT use `..` to ignore fields you do not want to deal with.
144        // Reject them with a clear NotImplemented error.
145        let ColumnDef {
146            name,
147            data_type,
148            collation,
149            options,
150            ..
151        } = column;
152
153        let data_type = data_type
154            .clone()
155            .ok_or_else(|| ErrorCode::InvalidInputSyntax("data type is not specified".into()))?;
156        if let Some(collation) = collation {
157            // PostgreSQL will limit the datatypes that collate can work on.
158            // https://www.postgresql.org/docs/16/collation.html#COLLATION-CONCEPTS
159            //   > The built-in collatable data types are `text`, `varchar`, and `char`.
160            //
161            // But we don't support real collation, we simply ignore it here.
162            if !["C", "POSIX"].contains(&collation.real_value().as_str()) {
163                bail_not_implemented!(
164                    "Collate collation other than `C` or `POSIX` is not implemented"
165                );
166            }
167
168            match data_type {
169                AstDataType::Text | AstDataType::Varchar | AstDataType::Char(_) => {}
170                _ => {
171                    return Err(ErrorCode::NotSupported(
172                        format!("{} is not a collatable data type", data_type),
173                        "The only built-in collatable data types are `varchar`, please check your type".into(),
174                    ).into());
175                }
176            }
177        }
178
179        if !is_for_drop_table_connector {
180            // additional column name may have prefix _rw
181            // When converting dropping the connector from table, the additional columns are converted to normal columns and keep the original name.
182            // Under this case, we loosen the check for _rw prefix.
183            check_column_name_not_reserved(&name.real_value())?;
184        }
185
186        let nullable: bool = !options
187            .iter()
188            .any(|def| matches!(def.option, ColumnOption::NotNull));
189
190        columns.push(ColumnCatalog {
191            column_desc: ColumnDesc {
192                data_type: bind_data_type(&data_type)?,
193                column_id: ColumnId::placeholder(),
194                name: name.real_value(),
195                generated_or_default_column: None,
196                description: None,
197                additional_column: AdditionalColumn { column_type: None },
198                version: ColumnDescVersion::LATEST,
199                system_column: None,
200                nullable,
201            },
202            is_hidden: false,
203        });
204    }
205
206    Ok(columns)
207}
208
209fn check_generated_column_constraints(
210    column_name: &String,
211    column_id: ColumnId,
212    expr: &ExprImpl,
213    column_catalogs: &[ColumnCatalog],
214    generated_column_names: &[String],
215    pk_column_ids: &[ColumnId],
216) -> Result<()> {
217    let input_refs = expr.collect_input_refs(column_catalogs.len());
218    for idx in input_refs.ones() {
219        let referred_generated_column = &column_catalogs[idx].column_desc.name;
220        if generated_column_names
221            .iter()
222            .any(|c| c == referred_generated_column)
223        {
224            return Err(ErrorCode::BindError(format!(
225                "A generated column cannot reference another generated column. \
226                Generated column \"{}\" references generated column \"{}\"",
227                column_name, referred_generated_column
228            ))
229            .into());
230        }
231    }
232
233    if pk_column_ids.contains(&column_id) && expr.is_impure() {
234        return Err(ErrorCode::BindError(format!(
235            "Generated columns with impure expressions should not be part of the primary key. \
236            Here column \"{}\" is defined as part of the primary key.",
237            column_name
238        ))
239        .into());
240    }
241
242    Ok(())
243}
244
245/// Binds constraints that can be only specified in column definitions,
246/// currently generated columns and default columns.
247pub fn bind_sql_column_constraints(
248    session: &SessionImpl,
249    table_name: String,
250    column_catalogs: &mut [ColumnCatalog],
251    columns: &[ColumnDef],
252    pk_column_ids: &[ColumnId],
253) -> Result<()> {
254    let generated_column_names = {
255        let mut names = vec![];
256        for column in columns {
257            for option_def in &column.options {
258                if let ColumnOption::GeneratedColumns(_) = option_def.option {
259                    names.push(column.name.real_value());
260                    break;
261                }
262            }
263        }
264        names
265    };
266
267    let mut binder = Binder::new_for_ddl(session);
268    binder.bind_columns_to_context(table_name, column_catalogs)?;
269
270    for column in columns {
271        let Some(idx) = column_catalogs
272            .iter()
273            .position(|c| c.name() == column.name.real_value())
274        else {
275            // It's possible that we don't follow the user defined columns in SQL but take the
276            // ones resolved from the source, thus missing some columns. Simply ignore them.
277            continue;
278        };
279
280        for option_def in &column.options {
281            match &option_def.option {
282                ColumnOption::GeneratedColumns(expr) => {
283                    binder.set_clause(Some(Clause::GeneratedColumn));
284
285                    let expr_impl = binder.bind_expr(expr).with_context(|| {
286                        format!(
287                            "failed to bind the expression in generated column \"{}\"",
288                            column.name.real_value()
289                        )
290                    })?;
291
292                    check_generated_column_constraints(
293                        &column.name.real_value(),
294                        column_catalogs[idx].column_id(),
295                        &expr_impl,
296                        column_catalogs,
297                        &generated_column_names,
298                        pk_column_ids,
299                    )?;
300
301                    column_catalogs[idx].column_desc.generated_or_default_column = Some(
302                        GeneratedOrDefaultColumn::GeneratedColumn(GeneratedColumnDesc {
303                            expr: Some(expr_impl.to_expr_proto()),
304                        }),
305                    );
306                    binder.set_clause(None);
307                }
308                ColumnOption::DefaultValue(expr) => {
309                    let expr_impl = binder
310                        .bind_expr(expr)?
311                        .cast_assign(column_catalogs[idx].data_type())?;
312
313                    // Rewrite expressions to evaluate a snapshot value, used for missing values in the case of
314                    // schema change.
315                    //
316                    // TODO: Currently we don't support impure expressions other than `now()` (like `random()`),
317                    // so the rewritten expression should almost always be pure and we directly call `fold_const`
318                    // here. Actually we do not require purity of the expression here since we're only to get a
319                    // snapshot value.
320                    let rewritten_expr_impl = session
321                        .pinned_snapshot()
322                        .inline_now_proc_time()
323                        .rewrite_expr(expr_impl.clone());
324
325                    if let Some(snapshot_value) = rewritten_expr_impl.try_fold_const() {
326                        let snapshot_value = snapshot_value?;
327
328                        column_catalogs[idx].column_desc.generated_or_default_column =
329                            Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
330                                snapshot_value: Some(snapshot_value.to_protobuf()),
331                                expr: Some(expr_impl.to_expr_proto()),
332                                // persist the original expression
333                            }));
334                    } else {
335                        return Err(ErrorCode::BindError(format!(
336                            "Default expression used in column `{}` cannot be evaluated. \
337                            Use generated columns instead if you mean to reference other columns.",
338                            column.name
339                        ))
340                        .into());
341                    }
342                }
343                ColumnOption::DefaultValueInternal { persisted, expr: _ } => {
344                    // When a `DEFAULT INTERNAL` is used internally for schema change, the persisted value
345                    // should already be set during purifcation. So if we encounter an empty value here, it
346                    // means the user has specified it explicitly in the SQL statement, typically by
347                    // directly copying the result of `SHOW CREATE TABLE` and executing it.
348                    if persisted.is_empty() {
349                        bail_bind_error!(
350                            "DEFAULT INTERNAL is only used for internal purposes, \
351                             please specify a concrete default value"
352                        );
353                    }
354
355                    let desc = DefaultColumnDesc::decode(&**persisted)
356                        .expect("failed to decode persisted `DefaultColumnDesc`");
357
358                    column_catalogs[idx].column_desc.generated_or_default_column =
359                        Some(GeneratedOrDefaultColumn::DefaultColumn(desc));
360                }
361                _ => {}
362            }
363        }
364    }
365    Ok(())
366}
367
368/// Currently we only support Primary key table constraint, so just return pk names if it exists
369pub fn bind_table_constraints(table_constraints: &[TableConstraint]) -> Result<Vec<String>> {
370    let mut pk_column_names = vec![];
371
372    for constraint in table_constraints {
373        match constraint {
374            TableConstraint::Unique {
375                name: _,
376                columns,
377                is_primary: true,
378            } => {
379                if !pk_column_names.is_empty() {
380                    return Err(multiple_pk_definition_err());
381                }
382                pk_column_names = columns.iter().map(|c| c.real_value()).collect_vec();
383            }
384            _ => bail_not_implemented!("table constraint \"{}\"", constraint),
385        }
386    }
387    Ok(pk_column_names)
388}
389
390pub fn bind_sql_pk_names(
391    columns_defs: &[ColumnDef],
392    pk_names_from_table_constraints: Vec<String>,
393) -> Result<Vec<String>> {
394    let mut pk_column_names = pk_names_from_table_constraints;
395
396    for column in columns_defs {
397        for option_def in &column.options {
398            if let ColumnOption::Unique { is_primary: true } = option_def.option {
399                if !pk_column_names.is_empty() {
400                    return Err(multiple_pk_definition_err());
401                }
402                pk_column_names.push(column.name.real_value());
403            };
404        }
405    }
406
407    Ok(pk_column_names)
408}
409
410fn multiple_pk_definition_err() -> RwError {
411    ErrorCode::BindError("multiple primary keys are not allowed".into()).into()
412}
413
414/// Binds primary keys defined in SQL.
415///
416/// It returns the columns together with `pk_column_ids`, and an optional row id column index if
417/// added.
418pub fn bind_pk_and_row_id_on_relation(
419    mut columns: Vec<ColumnCatalog>,
420    pk_names: Vec<String>,
421    must_need_pk: bool,
422) -> Result<(Vec<ColumnCatalog>, Vec<ColumnId>, Option<usize>)> {
423    for c in &columns {
424        assert!(c.column_id() != ColumnId::placeholder());
425    }
426
427    // Mapping from column name to column id.
428    let name_to_id = columns
429        .iter()
430        .map(|c| (c.name(), c.column_id()))
431        .collect::<HashMap<_, _>>();
432
433    let mut pk_column_ids: Vec<_> = pk_names
434        .iter()
435        .map(|name| {
436            name_to_id.get(name.as_str()).copied().ok_or_else(|| {
437                ErrorCode::BindError(format!("column \"{name}\" named in key does not exist"))
438            })
439        })
440        .try_collect()?;
441
442    // Add `_row_id` column if `pk_column_ids` is empty and must_need_pk
443    let need_row_id = pk_column_ids.is_empty() && must_need_pk;
444
445    let row_id_index = need_row_id.then(|| {
446        let column = ColumnCatalog::row_id_column();
447        let index = columns.len();
448        pk_column_ids = vec![column.column_id()];
449        columns.push(column);
450        index
451    });
452
453    if let Some(col) = columns.iter().map(|c| c.name()).duplicates().next() {
454        Err(ErrorCode::InvalidInputSyntax(format!(
455            "column \"{col}\" specified more than once"
456        )))?;
457    }
458
459    Ok((columns, pk_column_ids, row_id_index))
460}
461
462/// `gen_create_table_plan_with_source` generates the plan for creating a table with an external
463/// stream source.
464#[allow(clippy::too_many_arguments)]
465pub(crate) async fn gen_create_table_plan_with_source(
466    mut handler_args: HandlerArgs,
467    explain_options: ExplainOptions,
468    table_name: ObjectName,
469    column_defs: Vec<ColumnDef>,
470    wildcard_idx: Option<usize>,
471    constraints: Vec<TableConstraint>,
472    format_encode: FormatEncodeOptions,
473    source_watermarks: Vec<SourceWatermark>,
474    mut col_id_gen: ColumnIdGenerator,
475    include_column_options: IncludeOption,
476    props: CreateTableProps,
477    sql_column_strategy: SqlColumnStrategy,
478) -> Result<(PlanRef, Option<SourceCatalog>, TableCatalog)> {
479    if props.append_only
480        && format_encode.format != Format::Plain
481        && format_encode.format != Format::Native
482    {
483        return Err(ErrorCode::BindError(format!(
484            "Append only table does not support format {}.",
485            format_encode.format
486        ))
487        .into());
488    }
489
490    let session = &handler_args.session;
491    let (with_properties, ingest_mode) =
492        bind_connector_props(&handler_args, &format_encode, false)?;
493    if with_properties.is_shareable_cdc_connector() {
494        generated_columns_check_for_cdc_table(&column_defs)?;
495        not_null_check_for_cdc_table(&wildcard_idx, &column_defs)?;
496    } else if column_defs.iter().any(|col| {
497        col.options
498            .iter()
499            .any(|def| matches!(def.option, ColumnOption::NotNull))
500    }) {
501        // if non-cdc source
502        notice_to_user(
503            "The table contains columns with NOT NULL constraints. Any rows from upstream violating the constraints will be ignored silently.",
504        );
505    }
506
507    let db_name: &str = &session.database();
508    let (schema_name, _) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
509    // Check the CREATE privilege before `bind_columns_from_source`, which may fetch from
510    // user-supplied URLs (schema registry, schema file).
511    session.get_database_and_schema_id_for_create(schema_name.clone())?;
512
513    // TODO: omit this step if `sql_column_strategy` is `Follow`.
514    let (columns_from_resolve_source, source_info) = bind_columns_from_source(
515        session,
516        &format_encode,
517        Either::Left(&with_properties),
518        CreateSourceType::Table,
519    )
520    .await?;
521
522    let overwrite_options = OverwriteOptions::new(&mut handler_args)?;
523    let rate_limit = overwrite_options.source_rate_limit;
524    let source = bind_create_source_or_table_with_connector(
525        handler_args.clone(),
526        table_name,
527        format_encode,
528        with_properties,
529        &column_defs,
530        constraints,
531        wildcard_idx,
532        source_watermarks,
533        columns_from_resolve_source,
534        source_info,
535        include_column_options,
536        &mut col_id_gen,
537        CreateSourceType::Table,
538        rate_limit,
539        sql_column_strategy,
540        ingest_mode,
541    )
542    .await?;
543
544    let context = OptimizerContext::new(handler_args, explain_options)?;
545
546    let (plan, table) = gen_table_plan_with_source(
547        context.into(),
548        schema_name,
549        source.clone(),
550        col_id_gen.into_version(),
551        props,
552    )?;
553
554    Ok((plan, Some(source), table))
555}
556
557/// `gen_create_table_plan` generates the plan for creating a table without an external stream
558/// source.
559#[allow(clippy::too_many_arguments)]
560pub(crate) fn gen_create_table_plan(
561    context: OptimizerContext,
562    table_name: ObjectName,
563    column_defs: Vec<ColumnDef>,
564    constraints: Vec<TableConstraint>,
565    mut col_id_gen: ColumnIdGenerator,
566    source_watermarks: Vec<SourceWatermark>,
567    props: CreateTableProps,
568    is_for_replace_plan: bool,
569) -> Result<(PlanRef, TableCatalog)> {
570    let mut columns = bind_sql_columns(&column_defs, is_for_replace_plan)?;
571    for c in &mut columns {
572        col_id_gen.generate(c)?;
573    }
574
575    let (_, secret_refs, connection_refs) = context.with_options().clone().into_parts();
576    if !secret_refs.is_empty() || !connection_refs.is_empty() {
577        return Err(crate::error::ErrorCode::InvalidParameterValue("Secret reference and Connection reference are not allowed in options when creating table without external source".to_owned()).into());
578    }
579
580    gen_create_table_plan_without_source(
581        context,
582        table_name,
583        columns,
584        column_defs,
585        constraints,
586        source_watermarks,
587        col_id_gen.into_version(),
588        props,
589    )
590}
591
592#[allow(clippy::too_many_arguments)]
593pub(crate) fn gen_create_table_plan_without_source(
594    context: OptimizerContext,
595    table_name: ObjectName,
596    columns: Vec<ColumnCatalog>,
597    column_defs: Vec<ColumnDef>,
598    constraints: Vec<TableConstraint>,
599    source_watermarks: Vec<SourceWatermark>,
600    version: TableVersion,
601    props: CreateTableProps,
602) -> Result<(PlanRef, TableCatalog)> {
603    // XXX: Why not bind outside?
604    let pk_names = bind_sql_pk_names(&column_defs, bind_table_constraints(&constraints)?)?;
605    let (mut columns, pk_column_ids, row_id_index) =
606        bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
607
608    let watermark_descs = bind_source_watermark(
609        context.session_ctx(),
610        table_name.real_value(),
611        source_watermarks,
612        &columns,
613    )?;
614
615    bind_sql_column_constraints(
616        context.session_ctx(),
617        table_name.real_value(),
618        &mut columns,
619        &column_defs,
620        &pk_column_ids,
621    )?;
622    let session = context.session_ctx().clone();
623
624    let db_name = &session.database();
625    let (schema_name, table_name) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
626
627    let info = CreateTableInfo {
628        columns,
629        pk_column_ids,
630        row_id_index,
631        watermark_descs,
632        source_catalog: None,
633        version,
634    };
635
636    gen_table_plan_inner(context.into(), schema_name, table_name, info, props)
637}
638
639fn gen_table_plan_with_source(
640    context: OptimizerContextRef,
641    schema_name: Option<String>,
642    source_catalog: SourceCatalog,
643    version: TableVersion,
644    props: CreateTableProps,
645) -> Result<(PlanRef, TableCatalog)> {
646    let table_name = source_catalog.name.clone();
647
648    let info = CreateTableInfo {
649        columns: source_catalog.columns.clone(),
650        pk_column_ids: source_catalog.pk_col_ids.clone(),
651        row_id_index: source_catalog.row_id_index,
652        watermark_descs: source_catalog.watermark_descs.clone(),
653        source_catalog: Some(source_catalog),
654        version,
655    };
656
657    gen_table_plan_inner(context, schema_name, table_name, info, props)
658}
659
660/// On-conflict behavior either from user input or existing table catalog.
661#[derive(Clone, Copy)]
662pub enum EitherOnConflict {
663    Ast(Option<OnConflict>),
664    Resolved(ConflictBehavior),
665}
666
667impl From<Option<OnConflict>> for EitherOnConflict {
668    fn from(v: Option<OnConflict>) -> Self {
669        Self::Ast(v)
670    }
671}
672
673impl From<ConflictBehavior> for EitherOnConflict {
674    fn from(v: ConflictBehavior) -> Self {
675        Self::Resolved(v)
676    }
677}
678
679impl EitherOnConflict {
680    /// Resolves the conflict behavior based on the given information.
681    pub fn to_behavior(self, append_only: bool, row_id_as_pk: bool) -> Result<ConflictBehavior> {
682        let conflict_behavior = match self {
683            EitherOnConflict::Ast(on_conflict) => {
684                if append_only {
685                    if row_id_as_pk {
686                        // Primary key will be generated, no conflict check needed.
687                        ConflictBehavior::NoCheck
688                    } else {
689                        // User defined PK on append-only table, enforce `DO NOTHING`.
690                        if let Some(on_conflict) = on_conflict
691                            && on_conflict != OnConflict::Nothing
692                        {
693                            return Err(ErrorCode::InvalidInputSyntax(
694                                "When PRIMARY KEY constraint applied to an APPEND ONLY table, \
695                                     the ON CONFLICT behavior must be DO NOTHING."
696                                    .to_owned(),
697                            )
698                            .into());
699                        }
700                        ConflictBehavior::IgnoreConflict
701                    }
702                } else {
703                    // Default to `UPDATE FULL` for non-append-only tables.
704                    match on_conflict.unwrap_or(OnConflict::UpdateFull) {
705                        OnConflict::UpdateFull => ConflictBehavior::Overwrite,
706                        OnConflict::Nothing => ConflictBehavior::IgnoreConflict,
707                        OnConflict::UpdateIfNotNull => ConflictBehavior::DoUpdateIfNotNull,
708                    }
709                }
710            }
711            EitherOnConflict::Resolved(b) => b,
712        };
713
714        Ok(conflict_behavior)
715    }
716}
717
718/// Arguments of the functions that generate a table plan, part 1.
719///
720/// Compared to [`CreateTableProps`], this struct contains fields that need some work of binding
721/// or resolving based on the user input.
722pub struct CreateTableInfo {
723    pub columns: Vec<ColumnCatalog>,
724    pub pk_column_ids: Vec<ColumnId>,
725    pub row_id_index: Option<usize>,
726    pub watermark_descs: Vec<WatermarkDesc>,
727    pub source_catalog: Option<SourceCatalog>,
728    pub version: TableVersion,
729}
730
731/// Arguments of the functions that generate a table plan, part 2.
732///
733/// Compared to [`CreateTableInfo`], this struct contains fields that can be (relatively) simply
734/// obtained from the input or the context.
735pub struct CreateTableProps {
736    pub definition: String,
737    pub append_only: bool,
738    pub on_conflict: EitherOnConflict,
739    pub with_version_columns: Vec<String>,
740    pub webhook_info: Option<PbWebhookSourceInfo>,
741    pub engine: Engine,
742}
743
744#[allow(clippy::too_many_arguments)]
745fn gen_table_plan_inner(
746    context: OptimizerContextRef,
747    schema_name: Option<String>,
748    table_name: String,
749    info: CreateTableInfo,
750    props: CreateTableProps,
751) -> Result<(PlanRef, TableCatalog)> {
752    let CreateTableInfo {
753        ref columns,
754        row_id_index,
755        ref watermark_descs,
756        ref source_catalog,
757        ..
758    } = info;
759    let CreateTableProps { append_only, .. } = props;
760
761    let (database_id, schema_id) = context
762        .session_ctx()
763        .get_database_and_schema_id_for_create(schema_name)?;
764
765    let session = context.session_ctx().clone();
766    let retention_seconds = context.with_options().retention_seconds();
767
768    let source_node: LogicalPlanRef = LogicalSource::new(
769        source_catalog.clone().map(Rc::new),
770        columns.clone(),
771        row_id_index,
772        SourceNodeKind::CreateTable,
773        context.clone(),
774        None,
775    )?
776    .into();
777
778    let required_cols = FixedBitSet::with_capacity(columns.len());
779    let plan_root = PlanRoot::new_with_logical_plan(
780        source_node,
781        RequiredDist::Any,
782        Order::any(),
783        required_cols,
784        vec![],
785    );
786
787    let has_non_ttl_watermark = watermark_descs.iter().any(|d| !d.with_ttl);
788
789    if !append_only && has_non_ttl_watermark {
790        return Err(ErrorCode::NotSupported(
791            "Defining watermarks on table requires the table to be append only.".to_owned(),
792            "Use the key words `APPEND ONLY`".to_owned(),
793        )
794        .into());
795    }
796
797    if !append_only && retention_seconds.is_some() {
798        if session
799            .config()
800            .unsafe_enable_storage_retention_for_non_append_only_tables()
801        {
802            tracing::warn!(
803                "Storage retention is enabled for non-append-only table {}. This may lead to stream inconsistency.",
804                table_name
805            );
806            const NOTICE: &str = "Storage retention is enabled for non-append-only table. \
807                                  This may lead to stream inconsistency and unrecoverable \
808                                  node failure if there is any row INSERT/UPDATE/DELETE operation \
809                                  corresponding to the TTLed primary keys";
810            session.notice_to_user(NOTICE);
811        } else {
812            return Err(ErrorCode::NotSupported(
813                "Defining retention seconds on table requires the table to be append only."
814                    .to_owned(),
815                "Use the key words `APPEND ONLY`".to_owned(),
816            )
817            .into());
818        }
819    }
820
821    let materialize =
822        plan_root.gen_table_plan(context, table_name, database_id, schema_id, info, props)?;
823
824    let mut table = materialize.table().clone();
825    table.owner = session.user_id();
826
827    Ok((
828        ensure_sync_log_store_fragment_root(materialize.into()),
829        table,
830    ))
831}
832
833/// Generate stream plan for cdc table based on shared source.
834/// In replace workflow, the `table_id` is the id of the table to be replaced
835/// in create table workflow, the `table_id` is a placeholder will be filled in the Meta
836#[allow(clippy::too_many_arguments)]
837pub(crate) fn gen_create_table_plan_for_cdc_table(
838    context: OptimizerContextRef,
839    source: Arc<SourceCatalog>,
840    external_table_name: String,
841    column_defs: Vec<ColumnDef>,
842    source_watermarks: Vec<SourceWatermark>,
843    mut columns: Vec<ColumnCatalog>,
844    pk_names: Vec<String>,
845    pk_comparisons: Vec<CdcKeyComparison>,
846    cdc_with_options: WithOptionsSecResolved,
847    mut col_id_gen: ColumnIdGenerator,
848    on_conflict: Option<OnConflict>,
849    with_version_columns: Vec<String>,
850    include_column_options: IncludeOption,
851    table_name: ObjectName,
852    resolved_table_name: String, // table name without schema prefix
853    database_id: DatabaseId,
854    schema_id: SchemaId,
855    table_id: TableId,
856    engine: Engine,
857) -> Result<(PlanRef, TableCatalog)> {
858    let session = context.session_ctx().clone();
859
860    // append additional columns to the end
861    handle_addition_columns(
862        None,
863        &cdc_with_options,
864        include_column_options,
865        &mut columns,
866        true,
867    )?;
868
869    for c in &mut columns {
870        col_id_gen.generate(c)?;
871    }
872
873    let (mut columns, pk_column_ids, _row_id_index) =
874        bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
875
876    let watermark_descs = bind_source_watermark(
877        context.session_ctx(),
878        table_name.real_value(),
879        source_watermarks,
880        &columns,
881    )?;
882
883    // NOTES: In auto schema change, default value is not provided in column definition.
884    bind_sql_column_constraints(
885        context.session_ctx(),
886        table_name.real_value(),
887        &mut columns,
888        &column_defs,
889        &pk_column_ids,
890    )?;
891
892    let definition = context.normalized_sql().to_owned();
893
894    let pk_column_indices = {
895        let mut id_to_idx = HashMap::new();
896        columns.iter().enumerate().for_each(|(idx, c)| {
897            id_to_idx.insert(c.column_id(), idx);
898        });
899        // pk column id must exist in table columns.
900        pk_column_ids
901            .iter()
902            .map(|c| id_to_idx.get(c).copied().unwrap())
903            .collect_vec()
904    };
905    let table_pk = pk_column_indices
906        .iter()
907        .map(|idx| ColumnOrder::new(*idx, OrderType::ascending()))
908        .collect();
909
910    let (options, secret_refs) = cdc_with_options.into_parts();
911
912    let non_generated_column_descs = columns
913        .iter()
914        .filter(|&c| !c.is_generated())
915        .map(|c| c.column_desc.clone())
916        .collect_vec();
917    let non_generated_column_num = non_generated_column_descs.len();
918    let cdc_table_type = ExternalCdcTableType::from_properties(&options);
919    let canonical_name = dispatch_cdc_source_type!(
920        options.cdc_source_type(),
921        T,
922        T::canonical_name(&options),
923        unreachable!("CDC table without a CDC connector")
924    );
925    let cdc_table_desc = CdcTableDesc {
926        table_id,
927        source_id: source.id, // id of cdc source streaming job
928        external_table_name: external_table_name.clone(),
929        pk: table_pk,
930        pk_comparisons,
931        columns: non_generated_column_descs,
932        stream_key: pk_column_indices,
933        connect_properties: options,
934        secret_refs,
935    };
936
937    tracing::debug!(?cdc_table_desc, "create cdc table");
938    let options = build_cdc_scan_options_with_options(
939        context.with_options(),
940        cdc_table_desc.connect_properties.cdc_source_type(),
941    )?;
942
943    let logical_scan = LogicalCdcScan::create(
944        external_table_name,
945        Rc::new(cdc_table_desc),
946        context.clone(),
947        options,
948    );
949
950    let scan_node: LogicalPlanRef = logical_scan.into();
951    let required_cols = FixedBitSet::with_capacity(non_generated_column_num);
952    let plan_root = PlanRoot::new_with_logical_plan(
953        scan_node,
954        RequiredDist::Any,
955        Order::any(),
956        required_cols,
957        vec![],
958    );
959
960    let cdc_table_id = build_cdc_table_id(source.id, &canonical_name);
961    let materialize = plan_root.gen_table_plan(
962        context,
963        resolved_table_name,
964        database_id,
965        schema_id,
966        CreateTableInfo {
967            columns,
968            pk_column_ids,
969            row_id_index: None,
970            watermark_descs,
971            source_catalog: Some((*source).clone()),
972            version: col_id_gen.into_version(),
973        },
974        CreateTableProps {
975            definition,
976            append_only: false,
977            on_conflict: on_conflict.into(),
978            with_version_columns,
979            webhook_info: None,
980            engine,
981        },
982    )?;
983
984    let mut table = materialize.table().clone();
985    table.owner = session.user_id();
986    table.cdc_table_id = Some(cdc_table_id);
987    table.cdc_table_type = Some(cdc_table_type);
988    Ok((
989        ensure_sync_log_store_fragment_root(materialize.into()),
990        table,
991    ))
992}
993
994#[allow(clippy::too_many_arguments)]
995pub(super) async fn handle_create_table_plan(
996    handler_args: HandlerArgs,
997    explain_options: ExplainOptions,
998    format_encode: Option<FormatEncodeOptions>,
999    cdc_table_info: Option<CdcTableInfo>,
1000    table_name: &ObjectName,
1001    column_defs: Vec<ColumnDef>,
1002    wildcard_idx: Option<usize>,
1003    constraints: Vec<TableConstraint>,
1004    source_watermarks: Vec<SourceWatermark>,
1005    append_only: bool,
1006    on_conflict: Option<OnConflict>,
1007    with_version_columns: Vec<String>,
1008    include_column_options: IncludeOption,
1009    webhook_info: Option<WebhookSourceInfo>,
1010    engine: Engine,
1011) -> Result<(
1012    PlanRef,
1013    Option<SourceCatalog>,
1014    TableCatalog,
1015    TableJobType,
1016    Option<SourceId>,
1017)> {
1018    let col_id_gen = ColumnIdGenerator::new_initial();
1019    let format_encode = check_create_table_with_source(
1020        &handler_args.with_options,
1021        format_encode,
1022        &include_column_options,
1023        &cdc_table_info,
1024    )?;
1025    let webhook_info = webhook_info
1026        .map(|info| bind_webhook_info(&handler_args.session, &column_defs, info))
1027        .transpose()?;
1028
1029    let props = CreateTableProps {
1030        definition: handler_args.normalized_sql.clone(),
1031        append_only,
1032        on_conflict: on_conflict.into(),
1033        with_version_columns: with_version_columns.clone(),
1034        webhook_info,
1035        engine,
1036    };
1037
1038    let ((plan, source, table), job_type, shared_shource_id) = match (
1039        format_encode,
1040        cdc_table_info.as_ref(),
1041    ) {
1042        (Some(format_encode), None) => (
1043            gen_create_table_plan_with_source(
1044                handler_args,
1045                explain_options,
1046                table_name.clone(),
1047                column_defs,
1048                wildcard_idx,
1049                constraints,
1050                format_encode,
1051                source_watermarks,
1052                col_id_gen,
1053                include_column_options,
1054                props,
1055                SqlColumnStrategy::FollowChecked,
1056            )
1057            .await?,
1058            TableJobType::General,
1059            None,
1060        ),
1061        (None, None) => {
1062            let context = OptimizerContext::new(handler_args, explain_options)?;
1063            let (plan, table) = gen_create_table_plan(
1064                context,
1065                table_name.clone(),
1066                column_defs,
1067                constraints,
1068                col_id_gen,
1069                source_watermarks,
1070                props,
1071                false,
1072            )?;
1073
1074            ((plan, None, table), TableJobType::General, None)
1075        }
1076
1077        (None, Some(cdc_table)) => {
1078            sanity_check_for_table_on_cdc_source(
1079                append_only,
1080                &column_defs,
1081                &wildcard_idx,
1082                &constraints,
1083                &source_watermarks,
1084            )?;
1085
1086            generated_columns_check_for_cdc_table(&column_defs)?;
1087            not_null_check_for_cdc_table(&wildcard_idx, &column_defs)?;
1088
1089            let session = &handler_args.session;
1090            let db_name = &session.database();
1091            let user_name = &session.user_name();
1092            let search_path = session.config().search_path();
1093            let (schema_name, resolved_table_name) =
1094                Binder::resolve_schema_qualified_name(db_name, table_name)?;
1095            let (database_id, schema_id) =
1096                session.get_database_and_schema_id_for_create(schema_name.clone())?;
1097
1098            // cdc table cannot be append-only
1099            let (source_schema, source_name) =
1100                Binder::resolve_schema_qualified_name(db_name, &cdc_table.source_name)?;
1101
1102            let source = {
1103                let catalog_reader = session.env().catalog_reader().read_guard();
1104                let schema_path =
1105                    SchemaPath::new(source_schema.as_deref(), &search_path, user_name);
1106
1107                let (source, _) = catalog_reader.get_source_by_name(
1108                    db_name,
1109                    schema_path,
1110                    source_name.as_str(),
1111                )?;
1112                source.clone()
1113            };
1114            check_cdc_source_select_privilege(session, &source)?;
1115            let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1116                &source.with_properties,
1117                cdc_table.external_table_name.clone(),
1118            )?;
1119
1120            let excluded_columns = match handler_args.with_options.get(CDC_COLUMN_EXCLUDE_LIST) {
1121                Some(list) => {
1122                    let columns = parse_cdc_column_exclude_list(list);
1123                    if columns.is_empty() {
1124                        return Err(ErrorCode::InvalidInputSyntax(format!(
1125                            "`{CDC_COLUMN_EXCLUDE_LIST}` must list at least one column"
1126                        ))
1127                        .into());
1128                    }
1129                    if wildcard_idx.is_none() {
1130                        return Err(ErrorCode::InvalidInputSyntax(format!(
1131                            "`{CDC_COLUMN_EXCLUDE_LIST}` requires the `(*)` wildcard in the column list"
1132                        ))
1133                        .into());
1134                    }
1135                    columns
1136                }
1137                None => HashSet::new(),
1138            };
1139
1140            let (columns, pk_names, pk_comparisons) = match wildcard_idx {
1141                Some(_) => {
1142                    bind_cdc_table_schema_externally(cdc_with_options.clone(), &excluded_columns)
1143                        .await?
1144                }
1145                None => {
1146                    for column_def in &column_defs {
1147                        for option_def in &column_def.options {
1148                            if let ColumnOption::DefaultValue(_)
1149                            | ColumnOption::DefaultValueInternal { .. } = option_def.option
1150                            {
1151                                return Err(ErrorCode::NotSupported(
1152                                            "Default value for columns defined on the table created from a CDC source".into(),
1153                                            "Remove the default value expression in the column definitions".into(),
1154                                        )
1155                                            .into());
1156                            }
1157                        }
1158                    }
1159
1160                    let (columns, pk_names) =
1161                        bind_cdc_table_schema(&column_defs, &constraints, false)?;
1162                    let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1163                        cdc_with_options.clone(),
1164                        &pk_names,
1165                    ))
1166                    .await?;
1167
1168                    (columns, pk_names, pk_comparisons)
1169                }
1170            };
1171
1172            // CDC-table branch only: if Debezium column filters remove a PK column from the
1173            // change-event value, RisingWave reads NULL for that PK from the payload and silently
1174            // corrupts downstream (UPDATE turns into a fresh INSERT, DELETE no-ops). Plain
1175            // (non-CDC) tables don't hit this check.
1176            reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
1177
1178            let context: OptimizerContextRef =
1179                OptimizerContext::new(handler_args, explain_options)?.into();
1180            let shared_source_id = source.id;
1181            let (plan, table) = gen_create_table_plan_for_cdc_table(
1182                context,
1183                source,
1184                external_table_name,
1185                column_defs,
1186                source_watermarks,
1187                columns,
1188                pk_names,
1189                pk_comparisons,
1190                cdc_with_options,
1191                col_id_gen,
1192                on_conflict,
1193                with_version_columns,
1194                include_column_options,
1195                table_name.clone(),
1196                resolved_table_name,
1197                database_id,
1198                schema_id,
1199                TableId::placeholder(),
1200                engine,
1201            )?;
1202
1203            (
1204                (plan, None, table),
1205                TableJobType::SharedCdcSource,
1206                Some(shared_source_id),
1207            )
1208        }
1209        (Some(_), Some(_)) => {
1210            return Err(ErrorCode::NotSupported(
1211                "Data format and encoding format doesn't apply to table created from a CDC source"
1212                    .into(),
1213                "Remove the FORMAT and ENCODE specification".into(),
1214            )
1215            .into());
1216        }
1217    };
1218    Ok((plan, source, table, job_type, shared_shource_id))
1219}
1220
1221// For both table from cdc source and table with cdc connector
1222fn generated_columns_check_for_cdc_table(columns: &Vec<ColumnDef>) -> Result<()> {
1223    let mut found_generated_column = false;
1224    for column in columns {
1225        let mut is_generated = false;
1226
1227        for option_def in &column.options {
1228            if let ColumnOption::GeneratedColumns(_) = option_def.option {
1229                is_generated = true;
1230                break;
1231            }
1232        }
1233
1234        if is_generated {
1235            found_generated_column = true;
1236        } else if found_generated_column {
1237            return Err(ErrorCode::NotSupported(
1238                "Non-generated column found after a generated column.".into(),
1239                "Ensure that all generated columns appear at the end of the cdc table definition."
1240                    .into(),
1241            )
1242            .into());
1243        }
1244    }
1245    Ok(())
1246}
1247
1248#[allow(clippy::too_many_arguments)]
1249pub async fn handle_create_table(
1250    handler_args: HandlerArgs,
1251    table_name: ObjectName,
1252    column_defs: Vec<ColumnDef>,
1253    wildcard_idx: Option<usize>,
1254    constraints: Vec<TableConstraint>,
1255    if_not_exists: bool,
1256    format_encode: Option<FormatEncodeOptions>,
1257    source_watermarks: Vec<SourceWatermark>,
1258    append_only: bool,
1259    on_conflict: Option<OnConflict>,
1260    with_version_columns: Vec<String>,
1261    cdc_table_info: Option<CdcTableInfo>,
1262    include_column_options: IncludeOption,
1263    webhook_info: Option<WebhookSourceInfo>,
1264    ast_engine: risingwave_sqlparser::ast::Engine,
1265) -> Result<RwPgResponse> {
1266    let session = handler_args.session.clone();
1267
1268    if append_only {
1269        session.notice_to_user("APPEND ONLY TABLE is currently an experimental feature.");
1270    }
1271
1272    session.check_cluster_limits().await?;
1273
1274    let engine = match ast_engine {
1275        risingwave_sqlparser::ast::Engine::Hummock => Engine::Hummock,
1276        risingwave_sqlparser::ast::Engine::Iceberg => Engine::Iceberg,
1277    };
1278
1279    if let Either::Right(resp) = session.check_relation_name_duplicated(
1280        table_name.clone(),
1281        StatementType::CREATE_TABLE,
1282        if_not_exists,
1283    )? {
1284        return Ok(resp);
1285    }
1286
1287    let (graph, source, hummock_table, job_type, shared_source_id) = {
1288        let (plan, source, table, job_type, shared_source_id) = Box::pin(handle_create_table_plan(
1289            handler_args.clone(),
1290            ExplainOptions::default(),
1291            format_encode,
1292            cdc_table_info,
1293            &table_name,
1294            column_defs.clone(),
1295            wildcard_idx,
1296            constraints.clone(),
1297            source_watermarks,
1298            append_only,
1299            on_conflict,
1300            with_version_columns,
1301            include_column_options,
1302            webhook_info,
1303            engine,
1304        ))
1305        .await?;
1306        tracing::trace!("table_plan: {:?}", plan.explain_to_string());
1307
1308        let graph = build_graph(plan, Some(GraphJobType::Table))?;
1309
1310        (graph, source, table, job_type, shared_source_id)
1311    };
1312
1313    tracing::trace!(
1314        "name={}, graph=\n{}",
1315        table_name,
1316        serde_json::to_string_pretty(&graph).unwrap()
1317    );
1318
1319    let dependencies = shared_source_id
1320        .map(|id| HashSet::from([id.as_object_id()]))
1321        .unwrap_or_default();
1322
1323    // Handle engine
1324    match engine {
1325        Engine::Hummock => {
1326            let catalog_writer = session.catalog_writer()?;
1327            let action = match job_type {
1328                TableJobType::SharedCdcSource => LongRunningNotificationAction::MonitorBackfillJob,
1329                _ => LongRunningNotificationAction::DiagnoseBarrierLatency,
1330            };
1331            execute_with_long_running_notification(
1332                catalog_writer.create_table(
1333                    source.map(|s| s.to_prost()),
1334                    hummock_table.to_prost(),
1335                    graph,
1336                    job_type,
1337                    if_not_exists,
1338                    dependencies,
1339                ),
1340                &session,
1341                "CREATE TABLE",
1342                action,
1343            )
1344            .await?;
1345        }
1346        Engine::Iceberg => {
1347            let hummock_table_name = hummock_table.name.clone();
1348            session.create_staging_table(hummock_table.clone());
1349            let res = Box::pin(create_iceberg_engine_table(
1350                session.clone(),
1351                handler_args,
1352                source.map(|s| s.to_prost()),
1353                hummock_table,
1354                graph,
1355                table_name,
1356                job_type,
1357                if_not_exists,
1358            ))
1359            .await;
1360            session.drop_staging_table(&hummock_table_name);
1361            res?
1362        }
1363    }
1364
1365    Ok(PgResponse::empty_result(StatementType::CREATE_TABLE))
1366}
1367
1368fn build_iceberg_engine_sink_options(
1369    mut sink_options: BTreeMap<String, String>,
1370    user_options: &WithOptions,
1371    table: &TableCatalog,
1372    primary_key: &[String],
1373) -> Result<BTreeMap<String, String>> {
1374    sink_options.extend(
1375        user_options
1376            .iter()
1377            .filter(|(key, _)| is_iceberg_engine_option(key))
1378            .map(|(key, value)| (key.clone(), value.clone())),
1379    );
1380
1381    sink_options
1382        .entry(ENABLE_COMPACTION.to_owned())
1383        .or_insert_with(|| "true".to_owned());
1384    sink_options.insert(
1385        "type".to_owned(),
1386        if table.append_only {
1387            "append-only"
1388        } else {
1389            "upsert"
1390        }
1391        .to_owned(),
1392    );
1393
1394    // Supply the table primary key while parsing the typed config. In PK-index mode the
1395    // planner derives it from the upstream stream key, so it is removed again below.
1396    if !table.append_only {
1397        sink_options.insert("primary_key".to_owned(), primary_key.join(","));
1398    }
1399
1400    sink_options.insert("create_table_if_not_exists".to_owned(), "true".to_owned());
1401    sink_options.insert("is_exactly_once".to_owned(), "true".to_owned());
1402
1403    let config = IcebergConfig::from_btreemap(sink_options.clone())?;
1404
1405    // Both merge-on-read and copy-on-write initialize an equality-delete writer for upserts.
1406    // Reject unsupported key types before creating the remote Iceberg table.
1407    if !table.append_only && !config.enable_pk_index {
1408        for pk in table.pk() {
1409            let column = &table.columns()[pk.column_index];
1410            let data_type = column.data_type();
1411            if data_type.is_composite()
1412                || matches!(data_type, DataType::Float32 | DataType::Float64)
1413            {
1414                return Err(ErrorCode::NotSupported(
1415                    format!(
1416                        "Iceberg engine table primary key column \"{}\" has unsupported type {} for equality deletes",
1417                        column.name(), data_type
1418                    ),
1419                    "Use non-floating-point scalar primary key columns.".to_owned(),
1420                )
1421                .into());
1422            }
1423        }
1424    }
1425
1426    // Engine tables own their Iceberg maintenance policy, so enable manifest rewrites by default
1427    // whenever the table format supports them. Keep V3 disabled until rewrites preserve row lineage.
1428    if config.table_format_version() < FormatVersion::V3 {
1429        sink_options
1430            .entry(ENABLE_MANIFEST_REWRITE.to_owned())
1431            .or_insert_with(|| "true".to_owned());
1432    }
1433
1434    if let Some(partition_by) = &config.partition_by {
1435        let mut partition_columns = vec![];
1436        for (column, _) in parse_partition_by_exprs(partition_by.clone())? {
1437            table
1438                .columns()
1439                .iter()
1440                .find(|col| col.name().eq_ignore_ascii_case(&column))
1441                .ok_or_else(|| {
1442                    ErrorCode::InvalidInputSyntax(format!(
1443                        "Partition source column does not exist in schema: {}",
1444                        column
1445                    ))
1446                })?;
1447
1448            partition_columns.push(column);
1449        }
1450
1451        ensure_partition_columns_are_prefix_of_primary_key(&partition_columns, primary_key)
1452            .map_err(|_| {
1453                ErrorCode::InvalidInputSyntax(
1454                    "The partition columns should be the prefix of the primary key".to_owned(),
1455                )
1456            })?;
1457    }
1458
1459    if let Some(order_key) = &config.order_key {
1460        validate_order_key_columns(order_key, table.columns().iter().map(|col| col.name()))
1461            .map_err(|err| ErrorCode::InvalidInputSyntax(err.to_report_string()))?;
1462    }
1463
1464    if config.enable_pk_index {
1465        sink_options.remove("primary_key");
1466    } else {
1467        sink_options.insert(AUTO_SCHEMA_CHANGE_KEY.to_owned(), "true".to_owned());
1468    }
1469
1470    Ok(sink_options)
1471}
1472
1473/// Iceberg table engine is composed of hummock table, iceberg sink and iceberg source.
1474///
1475/// 1. fetch iceberg engine options from the meta node. Or use iceberg engine connection provided by users.
1476/// 2. create a hummock table
1477/// 3. create an iceberg sink
1478/// 4. create an iceberg source
1479///
1480/// See <https://github.com/risingwavelabs/risingwave/issues/21586> for an architecture diagram.
1481#[allow(clippy::too_many_arguments)]
1482pub async fn create_iceberg_engine_table(
1483    session: Arc<SessionImpl>,
1484    handler_args: HandlerArgs,
1485    mut source: Option<PbSource>,
1486    table: TableCatalog,
1487    graph: StreamFragmentGraph,
1488    table_name: ObjectName,
1489    job_type: PbTableJobType,
1490    if_not_exists: bool,
1491) -> Result<()> {
1492    let rw_db_name = session
1493        .env()
1494        .catalog_reader()
1495        .read_guard()
1496        .get_database_by_id(table.database_id)?
1497        .name()
1498        .to_owned();
1499    let rw_schema_name = session
1500        .env()
1501        .catalog_reader()
1502        .read_guard()
1503        .get_schema_by_id(table.database_id, table.schema_id)?
1504        .name()
1505        .clone();
1506    let iceberg_catalog_name = rw_db_name.clone();
1507    let iceberg_database_name = rw_schema_name.clone();
1508    let iceberg_table_name = table_name.0.last().unwrap().real_value();
1509
1510    let iceberg_engine_connection: String = session.config().iceberg_engine_connection();
1511    let sink_decouple = session.config().sink_decouple();
1512    if matches!(sink_decouple, SinkDecouple::Disable) {
1513        bail!(
1514            "Iceberg engine table only supports with sink decouple, try `set sink_decouple = true` to resolve it"
1515        );
1516    }
1517
1518    let mut connection_ref = BTreeMap::new();
1519    let with_common = if iceberg_engine_connection.is_empty() {
1520        bail!("to use iceberg engine table, the variable `iceberg_engine_connection` must be set.");
1521    } else {
1522        let parts: Vec<&str> = iceberg_engine_connection.split('.').collect();
1523        assert_eq!(parts.len(), 2);
1524        let connection_catalog =
1525            session.get_connection_by_name(Some(parts[0].to_owned()), parts[1])?;
1526        if let ConnectionInfo::ConnectionParams(params) = &connection_catalog.info {
1527            if params.connection_type == ConnectionType::Iceberg as i32 {
1528                // With iceberg engine connection:
1529                connection_ref.insert(
1530                    "connection".to_owned(),
1531                    ConnectionRefValue {
1532                        connection_name: ObjectName::from(vec![
1533                            Ident::from(parts[0]),
1534                            Ident::from(parts[1]),
1535                        ]),
1536                    },
1537                );
1538
1539                let mut with_common = BTreeMap::new();
1540                with_common.insert("connector".to_owned(), "iceberg".to_owned());
1541                with_common.insert("database.name".to_owned(), iceberg_database_name);
1542                with_common.insert("table.name".to_owned(), iceberg_table_name);
1543
1544                let hosted_catalog = params
1545                    .properties
1546                    .get("hosted_catalog")
1547                    .map(|s| s.eq_ignore_ascii_case("true"))
1548                    .unwrap_or(false);
1549                if hosted_catalog {
1550                    let meta_client = session.env().meta_client();
1551                    let meta_store_endpoint = meta_client.get_meta_store_endpoint().await?;
1552
1553                    let meta_store_endpoint =
1554                        url::Url::parse(&meta_store_endpoint).map_err(|_| {
1555                            ErrorCode::InternalError(
1556                                "failed to parse the meta store endpoint".to_owned(),
1557                            )
1558                        })?;
1559                    let meta_store_backend = meta_store_endpoint.scheme().to_owned();
1560                    let meta_store_user = meta_store_endpoint.username().to_owned();
1561                    let meta_store_password = match meta_store_endpoint.password() {
1562                        Some(password) => percent_decode_str(password)
1563                            .decode_utf8()
1564                            .map_err(|_| {
1565                                ErrorCode::InternalError(
1566                                    "failed to parse password from meta store endpoint".to_owned(),
1567                                )
1568                            })?
1569                            .into_owned(),
1570                        None => "".to_owned(),
1571                    };
1572                    let meta_store_host = meta_store_endpoint
1573                        .host_str()
1574                        .ok_or_else(|| {
1575                            ErrorCode::InternalError(
1576                                "failed to parse host from meta store endpoint".to_owned(),
1577                            )
1578                        })?
1579                        .to_owned();
1580                    let meta_store_port = meta_store_endpoint.port().ok_or_else(|| {
1581                        ErrorCode::InternalError(
1582                            "failed to parse port from meta store endpoint".to_owned(),
1583                        )
1584                    })?;
1585                    let meta_store_database = meta_store_endpoint
1586                        .path()
1587                        .trim_start_matches('/')
1588                        .to_owned();
1589
1590                    let Ok(meta_backend) = MetaBackend::from_str(&meta_store_backend, true) else {
1591                        bail!("failed to parse meta backend: {}", meta_store_backend);
1592                    };
1593
1594                    let catalog_uri = match meta_backend {
1595                        MetaBackend::Postgres => {
1596                            format!(
1597                                "jdbc:postgresql://{}:{}/{}",
1598                                meta_store_host, meta_store_port, meta_store_database
1599                            )
1600                        }
1601                        MetaBackend::Mysql => {
1602                            format!(
1603                                "jdbc:mysql://{}:{}/{}",
1604                                meta_store_host, meta_store_port, meta_store_database
1605                            )
1606                        }
1607                        MetaBackend::Sqlite | MetaBackend::Sql | MetaBackend::Mem => {
1608                            bail!(
1609                                "Unsupported meta backend for iceberg engine table: {}",
1610                                meta_store_backend
1611                            );
1612                        }
1613                    };
1614
1615                    with_common.insert("catalog.type".to_owned(), "jdbc".to_owned());
1616                    with_common.insert("catalog.uri".to_owned(), catalog_uri);
1617                    with_common.insert("catalog.jdbc.user".to_owned(), meta_store_user);
1618                    with_common.insert("catalog.jdbc.password".to_owned(), meta_store_password);
1619                    with_common.insert("catalog.name".to_owned(), iceberg_catalog_name);
1620                }
1621
1622                with_common
1623            } else {
1624                return Err(RwError::from(ErrorCode::InvalidParameterValue(
1625                    "Only iceberg connection could be used in iceberg engine".to_owned(),
1626                )));
1627            }
1628        } else {
1629            return Err(RwError::from(ErrorCode::InvalidParameterValue(
1630                "Private Link Service has been deprecated. Please create a new connection instead."
1631                    .to_owned(),
1632            )));
1633        }
1634    };
1635
1636    // Iceberg sinks require a primary key, if none is provided, we will use the _row_id column
1637    // Fetch primary key from columns
1638    let mut pks = table
1639        .pk_column_names()
1640        .iter()
1641        .map(|c| c.to_string())
1642        .collect::<Vec<String>>();
1643
1644    // For the table without primary key. We will use `_row_id` as primary key.
1645    if pks.len() == 1 && pks[0].eq(ROW_ID_COLUMN_NAME) {
1646        pks = vec![RISINGWAVE_ICEBERG_ROW_ID.to_owned()];
1647    }
1648
1649    let sink_from = CreateSink::From(table_name.clone());
1650
1651    let mut sink_name = table_name.clone();
1652    *sink_name.0.last_mut().unwrap() = Ident::from(
1653        (ICEBERG_SINK_PREFIX.to_owned() + &sink_name.0.last().unwrap().real_value()).as_str(),
1654    );
1655    let create_sink_stmt = CreateSinkStatement {
1656        or_replace: false,
1657        if_not_exists: false,
1658        sink_name,
1659        with_properties: WithProperties(vec![]),
1660        sink_from,
1661        columns: vec![],
1662        emit_mode: None,
1663        sink_schema: None,
1664        into_table_name: None,
1665    };
1666
1667    let mut sink_handler_args = handler_args.clone();
1668
1669    let sink_with = build_iceberg_engine_sink_options(
1670        with_common.clone(),
1671        &handler_args.with_options,
1672        &table,
1673        &pks,
1674    )?;
1675
1676    if let Some(source) = source.as_mut() {
1677        source
1678            .with_properties
1679            .retain(|key, _| !is_iceberg_engine_option(key));
1680    }
1681
1682    // sink_with.insert(SINK_SNAPSHOT_OPTION.to_owned(), "false".to_owned());
1683    //
1684    // Note: in theory, we don't need to backfill from the table to the sink,
1685    // but we don't have atomic DDL now https://github.com/risingwavelabs/risingwave/issues/21863
1686    // so it may have potential data loss problem on the first barrier.
1687    //
1688    // For non-append-only table, we can always solve it by the initial sink with backfill, since
1689    // data will be present in hummock table.
1690    //
1691    // For append-only table, we need to be more careful.
1692    //
1693    // The possible cases for a table:
1694    // - For table without connector: it doesn't matter, since there's no data before the table is created
1695    // - For table with connector: we workarounded it by setting SOURCE_RATE_LIMIT to 0
1696    //   + If we support blocking DDL for table with connector, we need to be careful.
1697    // - For table with an upstream job: Specifically, CDC table from shared CDC source.
1698    //   + Data may come from both upstream connector, and CDC table backfill, so we need to pause both of them.
1699    //   + For now we don't support APPEND ONLY CDC table, so it's safe.
1700
1701    sink_handler_args.with_options =
1702        WithOptions::new(sink_with, Default::default(), connection_ref.clone());
1703    let SinkPlanContext {
1704        sink_plan,
1705        sink_catalog,
1706        ..
1707    } = gen_sink_plan(sink_handler_args, create_sink_stmt, None, true).await?;
1708    let sink_graph = build_graph(sink_plan, Some(GraphJobType::Sink))?;
1709
1710    let mut source_name = table_name.clone();
1711    *source_name.0.last_mut().unwrap() = Ident::from(
1712        (ICEBERG_SOURCE_PREFIX.to_owned() + &source_name.0.last().unwrap().real_value()).as_str(),
1713    );
1714    let create_source_stmt = CreateSourceStatement {
1715        temporary: false,
1716        if_not_exists: false,
1717        columns: vec![],
1718        source_name,
1719        wildcard_idx: Some(0),
1720        constraints: vec![],
1721        with_properties: WithProperties(vec![]),
1722        format_encode: CompatibleFormatEncode::V2(FormatEncodeOptions::none()),
1723        source_watermarks: vec![],
1724        include_column_options: vec![],
1725        cdc_table_info: None,
1726    };
1727
1728    let mut source_handler_args = handler_args.clone();
1729    let source_with = with_common;
1730    source_handler_args.with_options =
1731        WithOptions::new(source_with, Default::default(), connection_ref);
1732
1733    let overwrite_options = OverwriteOptions::new(&mut source_handler_args)?;
1734    let format_encode = create_source_stmt.format_encode.into_v2_with_warning();
1735    let (with_properties, ingest_mode) =
1736        bind_connector_props(&source_handler_args, &format_encode, true)?;
1737
1738    // Create iceberg sink table, used for iceberg source column binding. See `bind_columns_from_source_for_non_cdc` for more details.
1739    // TODO: We can derive the columns directly from table definition in the future, so that we don't need to pre-create the table catalog.
1740    let (iceberg_catalog, table_identifier, table_created) = {
1741        let sink_param = SinkParam::try_from_sink_catalog(sink_catalog.clone())?;
1742        let iceberg_sink = IcebergSink::try_from(sink_param)?;
1743        let table_created = iceberg_sink.create_table_if_not_exists().await?;
1744
1745        let iceberg_catalog = iceberg_sink.config.create_catalog().await?;
1746        let table_identifier = iceberg_sink.config.full_table_name()?;
1747        (iceberg_catalog, table_identifier, table_created)
1748    };
1749
1750    let create_source_type = CreateSourceType::for_newly_created(&session, &*with_properties);
1751    let (columns_from_resolve_source, source_info) = bind_columns_from_source(
1752        &session,
1753        &format_encode,
1754        Either::Left(&with_properties),
1755        create_source_type,
1756    )
1757    .await?;
1758    let mut col_id_gen = ColumnIdGenerator::new_initial();
1759
1760    let iceberg_source_catalog = bind_create_source_or_table_with_connector(
1761        source_handler_args,
1762        create_source_stmt.source_name,
1763        format_encode,
1764        with_properties,
1765        &create_source_stmt.columns,
1766        create_source_stmt.constraints,
1767        create_source_stmt.wildcard_idx,
1768        create_source_stmt.source_watermarks,
1769        columns_from_resolve_source,
1770        source_info,
1771        create_source_stmt.include_column_options,
1772        &mut col_id_gen,
1773        create_source_type,
1774        overwrite_options.source_rate_limit,
1775        SqlColumnStrategy::FollowChecked,
1776        ingest_mode,
1777    )
1778    .await?;
1779
1780    // before we create the table, ensure the JVM is initialized as we use jdbc catalog right now.
1781    // If JVM isn't initialized successfully, current not atomic ddl will result in a partially created iceberg engine table.
1782    let _ = Jvm::get_or_init()?;
1783
1784    let catalog_writer = session.catalog_writer()?;
1785    let action = match job_type {
1786        TableJobType::SharedCdcSource => LongRunningNotificationAction::MonitorBackfillJob,
1787        _ => LongRunningNotificationAction::DiagnoseBarrierLatency,
1788    };
1789    let res = execute_with_long_running_notification(
1790        catalog_writer.create_iceberg_table(
1791            PbTableJobInfo {
1792                source,
1793                table: Some(table.to_prost()),
1794                fragment_graph: Some(graph),
1795                job_type: job_type as _,
1796            },
1797            PbSinkJobInfo {
1798                sink: Some(sink_catalog.to_proto()),
1799                fragment_graph: Some(sink_graph),
1800            },
1801            iceberg_source_catalog.to_prost(),
1802            if_not_exists,
1803        ),
1804        &session,
1805        "CREATE TABLE",
1806        action,
1807    )
1808    .await;
1809
1810    if res.is_err() {
1811        // Only clean up an iceberg table this statement created; a pre-existing
1812        // external table must survive a failed CREATE.
1813        if table_created {
1814            let _ = iceberg_catalog
1815                .drop_table(&table_identifier)
1816                .await
1817                .inspect_err(|err| {
1818                    tracing::error!(
1819                        "failed to drop iceberg table {} after create iceberg engine table failed: {}",
1820                        table_identifier,
1821                        err.as_report()
1822                    );
1823                });
1824        }
1825        res?
1826    }
1827
1828    Ok(())
1829}
1830
1831pub fn check_create_table_with_source(
1832    with_options: &WithOptions,
1833    format_encode: Option<FormatEncodeOptions>,
1834    include_column_options: &IncludeOption,
1835    cdc_table_info: &Option<CdcTableInfo>,
1836) -> Result<Option<FormatEncodeOptions>> {
1837    // skip check for cdc table
1838    if cdc_table_info.is_some() {
1839        return Ok(format_encode);
1840    }
1841    let defined_source = with_options.is_source_connector();
1842
1843    if !include_column_options.is_empty() && !defined_source {
1844        return Err(ErrorCode::InvalidInputSyntax(
1845            "INCLUDE should be used with a connector".to_owned(),
1846        )
1847        .into());
1848    }
1849    if defined_source {
1850        format_encode.as_ref().ok_or_else(|| {
1851            ErrorCode::InvalidInputSyntax("Please specify a source schema using FORMAT".to_owned())
1852        })?;
1853    }
1854    Ok(format_encode)
1855}
1856
1857fn ensure_partition_columns_are_prefix_of_primary_key(
1858    partition_columns: &[String],
1859    primary_key_columns: &[String],
1860) -> std::result::Result<(), String> {
1861    if partition_columns.len() > primary_key_columns.len() {
1862        return Err("Partition columns cannot be longer than primary key columns.".to_owned());
1863    }
1864
1865    for (i, partition_col) in partition_columns.iter().enumerate() {
1866        if primary_key_columns.get(i) != Some(partition_col) {
1867            return Err(format!(
1868                "Partition column '{}' is not a prefix of the primary key.",
1869                partition_col
1870            ));
1871        }
1872    }
1873
1874    Ok(())
1875}
1876
1877#[allow(clippy::too_many_arguments)]
1878pub async fn generate_stream_graph_for_replace_table(
1879    _session: &Arc<SessionImpl>,
1880    table_name: ObjectName,
1881    original_catalog: &Arc<TableCatalog>,
1882    handler_args: HandlerArgs,
1883    statement: Statement,
1884    col_id_gen: ColumnIdGenerator,
1885    sql_column_strategy: SqlColumnStrategy,
1886) -> Result<(
1887    StreamFragmentGraph,
1888    TableCatalog,
1889    Option<SourceCatalog>,
1890    TableJobType,
1891)> {
1892    let Statement::CreateTable {
1893        columns,
1894        constraints,
1895        source_watermarks,
1896        append_only,
1897        on_conflict,
1898        with_version_columns,
1899        wildcard_idx,
1900        cdc_table_info,
1901        format_encode,
1902        include_column_options,
1903        engine,
1904        with_options,
1905        ..
1906    } = statement
1907    else {
1908        panic!("unexpected statement type: {:?}", statement);
1909    };
1910
1911    let format_encode = format_encode
1912        .clone()
1913        .map(|format_encode| format_encode.into_v2_with_warning());
1914
1915    let engine = match engine {
1916        risingwave_sqlparser::ast::Engine::Hummock => Engine::Hummock,
1917        risingwave_sqlparser::ast::Engine::Iceberg => Engine::Iceberg,
1918    };
1919
1920    let is_drop_connector =
1921        original_catalog.associated_source_id().is_some() && format_encode.is_none();
1922    if is_drop_connector {
1923        debug_assert!(
1924            include_column_options.is_empty()
1925                && with_options
1926                    .iter()
1927                    .all(|opt| opt.name.real_value().to_lowercase() != "connector")
1928        );
1929    }
1930
1931    let props = CreateTableProps {
1932        definition: handler_args.normalized_sql.clone(),
1933        append_only,
1934        on_conflict: on_conflict.into(),
1935        with_version_columns: with_version_columns
1936            .iter()
1937            .map(|col| col.real_value())
1938            .collect(),
1939        webhook_info: original_catalog.webhook_info.clone(),
1940        engine,
1941    };
1942
1943    let ((plan, mut source, mut table), job_type) = match (format_encode, cdc_table_info.as_ref()) {
1944        (Some(format_encode), None) => (
1945            gen_create_table_plan_with_source(
1946                handler_args,
1947                ExplainOptions::default(),
1948                table_name,
1949                columns,
1950                wildcard_idx,
1951                constraints,
1952                format_encode,
1953                source_watermarks,
1954                col_id_gen,
1955                include_column_options,
1956                props,
1957                sql_column_strategy,
1958            )
1959            .await?,
1960            TableJobType::General,
1961        ),
1962        (None, None) => {
1963            let context = OptimizerContext::from_handler_args(handler_args)?;
1964            let (plan, table) = gen_create_table_plan(
1965                context,
1966                table_name,
1967                columns,
1968                constraints,
1969                col_id_gen,
1970                source_watermarks,
1971                props,
1972                true,
1973            )?;
1974            ((plan, None, table), TableJobType::General)
1975        }
1976        (None, Some(cdc_table)) => {
1977            sanity_check_for_table_on_cdc_source(
1978                append_only,
1979                &columns,
1980                &wildcard_idx,
1981                &constraints,
1982                &source_watermarks,
1983            )?;
1984
1985            let session = &handler_args.session;
1986            let (source, resolved_table_name) =
1987                get_source_and_resolved_table_name(session, cdc_table.clone(), table_name.clone())?;
1988
1989            let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1990                &source.with_properties,
1991                cdc_table.external_table_name.clone(),
1992            )?;
1993
1994            let (column_catalogs, pk_names) = bind_cdc_table_schema(&columns, &constraints, true)?;
1995            let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1996                cdc_with_options.clone(),
1997                &pk_names,
1998            ))
1999            .await?;
2000
2001            // CDC-table branch only: see the comment at the symmetric call site in
2002            // `handle_create_table_plan`. Plain (non-CDC) tables don't hit this check.
2003            reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
2004
2005            let context: OptimizerContextRef =
2006                OptimizerContext::new(handler_args, ExplainOptions::default())?.into();
2007            let (plan, table) = gen_create_table_plan_for_cdc_table(
2008                context,
2009                source,
2010                external_table_name,
2011                columns,
2012                source_watermarks,
2013                column_catalogs,
2014                pk_names,
2015                pk_comparisons,
2016                cdc_with_options,
2017                col_id_gen,
2018                on_conflict,
2019                with_version_columns
2020                    .iter()
2021                    .map(|col| col.real_value())
2022                    .collect(),
2023                include_column_options,
2024                table_name,
2025                resolved_table_name,
2026                original_catalog.database_id,
2027                original_catalog.schema_id,
2028                original_catalog.id(),
2029                engine,
2030            )?;
2031
2032            ((plan, None, table), TableJobType::SharedCdcSource)
2033        }
2034        (Some(_), Some(_)) => {
2035            return Err(ErrorCode::NotSupported(
2036                "Data format and encoding format doesn't apply to table created from a CDC source"
2037                    .into(),
2038                "Remove the FORMAT and ENCODE specification".into(),
2039            )
2040            .into());
2041        }
2042    };
2043
2044    if table.pk_column_ids() != original_catalog.pk_column_ids() {
2045        Err(ErrorCode::InvalidInputSyntax(
2046            "alter primary key of table is not supported".to_owned(),
2047        ))?
2048    }
2049
2050    let graph = build_graph(plan, Some(GraphJobType::Table))?;
2051
2052    // Fill the original table ID.
2053    table.id = original_catalog.id();
2054    if !is_drop_connector && let Some(source_id) = original_catalog.associated_source_id() {
2055        table.associated_source_id = Some(source_id);
2056
2057        let source = source.as_mut().unwrap();
2058        source.id = source_id;
2059        source.associated_table_id = Some(table.id());
2060    }
2061
2062    Ok((graph, table, source, job_type))
2063}
2064
2065fn get_source_and_resolved_table_name(
2066    session: &Arc<SessionImpl>,
2067    cdc_table: CdcTableInfo,
2068    table_name: ObjectName,
2069) -> Result<(Arc<SourceCatalog>, String)> {
2070    let db_name = &session.database();
2071    let (_, resolved_table_name) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
2072
2073    let (source_schema, source_name) =
2074        Binder::resolve_schema_qualified_name(db_name, &cdc_table.source_name)?;
2075
2076    let source = {
2077        let catalog_reader = session.env().catalog_reader().read_guard();
2078        let schema_name = source_schema.unwrap_or(DEFAULT_SCHEMA_NAME.to_owned());
2079        let (source, _) = catalog_reader.get_source_by_name(
2080            db_name,
2081            SchemaPath::Name(schema_name.as_str()),
2082            source_name.as_str(),
2083        )?;
2084        source.clone()
2085    };
2086    check_cdc_source_select_privilege(session, &source)?;
2087
2088    Ok((source, resolved_table_name))
2089}
2090
2091pub(crate) fn check_cdc_source_select_privilege(
2092    session: &SessionImpl,
2093    source: &SourceCatalog,
2094) -> Result<()> {
2095    session.check_privileges(&[ObjectCheckItem::new(
2096        source.owner,
2097        AclMode::Select,
2098        source.name.clone(),
2099        source.id,
2100    )])
2101}
2102
2103// validate the webhook_info and also bind the webhook_info to protobuf
2104fn bind_webhook_info(
2105    session: &Arc<SessionImpl>,
2106    column_defs: &[ColumnDef],
2107    webhook_info: WebhookSourceInfo,
2108) -> Result<PbWebhookSourceInfo> {
2109    let WebhookSourceInfo {
2110        secret_ref,
2111        signature_expr,
2112        wait_for_persistence,
2113        is_batched,
2114    } = webhook_info;
2115
2116    for column in column_defs {
2117        for option_def in &column.options {
2118            match option_def.option {
2119                ColumnOption::Null => {}
2120                ColumnOption::GeneratedColumns(_) => {
2121                    return Err(ErrorCode::InvalidInputSyntax(
2122                        "generated columns are not supported for webhook tables".to_owned(),
2123                    )
2124                    .into());
2125                }
2126                ColumnOption::DefaultValue(_) | ColumnOption::DefaultValueInternal { .. } => {
2127                    return Err(ErrorCode::InvalidInputSyntax(
2128                        "default values are not supported for webhook tables".to_owned(),
2129                    )
2130                    .into());
2131                }
2132                ColumnOption::NotNull
2133                | ColumnOption::Unique { .. }
2134                | ColumnOption::ForeignKey { .. }
2135                | ColumnOption::Check(_)
2136                | ColumnOption::DialectSpecific(_) => {
2137                    return Err(ErrorCode::InvalidInputSyntax(
2138                        "only NULL column option is supported for webhook tables".to_owned(),
2139                    )
2140                    .into());
2141                }
2142            }
2143        }
2144    }
2145
2146    // validate secret_ref
2147    let (pb_secret_ref, secret_name) = if let Some(secret_ref) = secret_ref {
2148        let db_name = &session.database();
2149        let (schema_name, secret_name) =
2150            Binder::resolve_schema_qualified_name(db_name, &secret_ref.secret_name)?;
2151        let secret_catalog = session.get_secret_by_name(schema_name, &secret_name)?;
2152        (
2153            Some(PbSecretRef {
2154                secret_id: secret_catalog.id,
2155                ref_as: match secret_ref.ref_as {
2156                    SecretRefAsType::Text => PbRefAsType::Text,
2157                    SecretRefAsType::File => PbRefAsType::File,
2158                }
2159                .into(),
2160            }),
2161            Some(secret_name),
2162        )
2163    } else {
2164        (None, None)
2165    };
2166
2167    let signature_expr = if let Some(signature_expr) = signature_expr {
2168        let payload_name = if column_defs.len() == 1
2169            && column_defs[0].data_type.as_ref() == Some(&AstDataType::Jsonb)
2170        {
2171            column_defs[0].name.real_value()
2172        } else {
2173            WEBHOOK_PAYLOAD_FIELD_NAME.to_owned()
2174        };
2175        let secure_compare_context = SecureCompareContext {
2176            payload_name,
2177            secret_name,
2178        };
2179        let mut binder = Binder::new_for_ddl(session).with_secure_compare(secure_compare_context);
2180        let expr = binder.bind_expr(&signature_expr)?;
2181
2182        // validate expr, ensuring it is SECURE_COMPARE()
2183        if expr.as_function_call().is_none()
2184            || expr.as_function_call().unwrap().func_type()
2185                != crate::optimizer::plan_node::generic::ExprType::SecureCompare
2186        {
2187            return Err(ErrorCode::InvalidInputSyntax(
2188                "The signature verification function must be SECURE_COMPARE()".to_owned(),
2189            )
2190            .into());
2191        }
2192
2193        Some(expr.to_expr_proto())
2194    } else {
2195        session.notice_to_user(
2196            "VALIDATE clause is strongly recommended for safety or production usages",
2197        );
2198        None
2199    };
2200
2201    let pb_webhook_info = PbWebhookSourceInfo {
2202        secret_ref: pb_secret_ref,
2203        signature_expr,
2204        wait_for_persistence,
2205        is_batched,
2206    };
2207
2208    Ok(pb_webhook_info)
2209}
2210
2211#[cfg(test)]
2212mod tests {
2213    use risingwave_common::catalog::{
2214        DEFAULT_DATABASE_NAME, ROW_ID_COLUMN_NAME, RW_TIMESTAMP_COLUMN_NAME,
2215    };
2216    use risingwave_common::types::StructType;
2217
2218    use super::*;
2219    use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
2220
2221    #[tokio::test]
2222    async fn test_iceberg_engine_primary_key_types() {
2223        let frontend = LocalFrontend::new(Default::default()).await;
2224        for (primary_key, append_only, pk_index, supported) in [
2225            ("id, period", false, false, false),
2226            ("id, items", false, false, false),
2227            ("id, mapping", false, false, false),
2228            ("id, f", false, false, false),
2229            ("id, d", false, false, false),
2230            ("id", false, false, true),
2231            ("id, ts", false, false, true),
2232            ("", false, false, true),
2233            ("id, period", true, false, true),
2234            ("id, period", false, true, true),
2235        ] {
2236            let pk_clause = if primary_key.is_empty() {
2237                String::new()
2238            } else {
2239                format!(", PRIMARY KEY ({primary_key})")
2240            };
2241            frontend
2242                .run_sql(&format!(
2243                    "CREATE TABLE t (id INT, period STRUCT<start_ts TIMESTAMPTZ, end_ts TIMESTAMPTZ>, \
2244                     items INT[], mapping MAP(INT, INT), f REAL, d DOUBLE PRECISION, ts TIMESTAMPTZ \
2245                     {pk_clause}) {}",
2246                    if append_only { "APPEND ONLY" } else { "" }
2247                ))
2248                .await
2249                .unwrap();
2250            let session = frontend.session_ref();
2251            let table = session
2252                .env()
2253                .catalog_reader()
2254                .read_guard()
2255                .get_created_table_by_name(
2256                    DEFAULT_DATABASE_NAME,
2257                    SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2258                    "t",
2259                )
2260                .unwrap()
2261                .0
2262                .clone();
2263            let pks = table
2264                .pk_column_names()
2265                .into_iter()
2266                .map(str::to_owned)
2267                .collect_vec();
2268            for write_mode in ["merge-on-read", "copy-on-write"] {
2269                if write_mode == "copy-on-write" && (append_only || pk_index) {
2270                    continue;
2271                }
2272                let options = BTreeMap::from([
2273                    ("connector".to_owned(), "iceberg".to_owned()),
2274                    ("catalog.type".to_owned(), "storage".to_owned()),
2275                    (
2276                        "warehouse.path".to_owned(),
2277                        "s3://test/warehouse".to_owned(),
2278                    ),
2279                    ("database.name".to_owned(), "public".to_owned()),
2280                    ("table.name".to_owned(), "t".to_owned()),
2281                    ("write_mode".to_owned(), write_mode.to_owned()),
2282                    ("enable_pk_index".to_owned(), pk_index.to_string()),
2283                ]);
2284                let result = build_iceberg_engine_sink_options(
2285                    options,
2286                    &WithOptions::default(),
2287                    &table,
2288                    &pks,
2289                );
2290                if supported {
2291                    result.unwrap();
2292                } else {
2293                    let err = result.unwrap_err().to_report_string();
2294                    let column = primary_key.split(", ").last().unwrap();
2295                    assert!(
2296                        err.contains(&format!(
2297                            "Iceberg engine table primary key column \"{column}\" has unsupported type"
2298                        )),
2299                        "{write_mode}: {err}"
2300                    );
2301                }
2302            }
2303            frontend.run_sql("DROP TABLE t").await.unwrap();
2304        }
2305    }
2306
2307    #[tokio::test]
2308    async fn test_cdc_relations_require_select_privilege_on_source() {
2309        let frontend = LocalFrontend::new(Default::default()).await;
2310        // PostgreSQL CDC with an explicit schema avoids querying upstream PK metadata,
2311        // so this privilege test does not require an external database.
2312        frontend
2313            .run_sql(
2314                r#"
2315                CREATE SOURCE cdc_source WITH (
2316                    connector = 'postgres-cdc',
2317                    hostname = 'localhost',
2318                    port = '5432',
2319                    username = 'postgres',
2320                    password = '',
2321                    database.name = 'db'
2322                ) FORMAT PLAIN ENCODE JSON
2323                "#,
2324            )
2325            .await
2326            .unwrap();
2327        frontend.run_sql("CREATE USER cdc_user").await.unwrap();
2328        frontend
2329            .run_sql("GRANT CREATE ON SCHEMA public TO cdc_user")
2330            .await
2331            .unwrap();
2332
2333        let user_id = frontend
2334            .session_ref()
2335            .env()
2336            .user_info_reader()
2337            .read_guard()
2338            .get_user_by_name("cdc_user")
2339            .unwrap()
2340            .id;
2341        let user_session = frontend.session_user_ref(
2342            DEFAULT_DATABASE_NAME.to_owned(),
2343            "cdc_user".to_owned(),
2344            user_id,
2345        );
2346        let create_table =
2347            "CREATE TABLE cdc_table (id INT PRIMARY KEY) FROM cdc_source TABLE 'public.t'";
2348        let create_table_source = "CREATE SOURCE cdc_table_source (id INT PRIMARY KEY) \
2349             WITH (snapshot = 'false') FROM cdc_source TABLE 'public.t'";
2350
2351        for sql in [create_table, create_table_source] {
2352            let err = frontend
2353                .run_sql_with_session(user_session.clone(), sql)
2354                .await
2355                .unwrap_err();
2356            assert!(
2357                err.to_string()
2358                    .contains("permission denied for source \"cdc_source\": \"SELECT\""),
2359                "{err:?}"
2360            );
2361        }
2362
2363        frontend
2364            .run_sql("GRANT SELECT ON SOURCE cdc_source TO cdc_user")
2365            .await
2366            .unwrap();
2367        frontend
2368            .run_sql_with_session(user_session.clone(), create_table)
2369            .await
2370            .unwrap();
2371        frontend
2372            .run_sql_with_session(user_session.clone(), create_table_source)
2373            .await
2374            .unwrap();
2375
2376        let table_source = {
2377            let catalog_reader = frontend.session_ref().env().catalog_reader().read_guard();
2378            catalog_reader
2379                .get_source_by_name(
2380                    DEFAULT_DATABASE_NAME,
2381                    SchemaPath::Name("public"),
2382                    "cdc_table_source",
2383                )
2384                .unwrap()
2385                .0
2386                .clone()
2387        };
2388        assert_eq!(table_source.with_properties.len(), 2);
2389        assert_eq!(
2390            table_source.with_properties.get_connector().as_deref(),
2391            Some("postgres-cdc")
2392        );
2393        assert_eq!(
2394            table_source
2395                .with_properties
2396                .get("snapshot")
2397                .map(String::as_str),
2398            Some("false")
2399        );
2400        assert!(table_source.with_properties.as_secret().is_empty());
2401        // The descriptor carries the resolved connect properties so that meta can validate it
2402        // against the upstream table; meta clears them before persisting the catalog, which is
2403        // covered by `test_create_cdc_table_source_dependency`.
2404        let external_table = table_source.info.external_table.as_ref().unwrap();
2405        assert!(!external_table.connect_properties.is_empty());
2406
2407        let err = frontend
2408            .run_sql_with_session(
2409                user_session.clone(),
2410                "SELECT * FROM postgres_query('cdc_table_source', 'SELECT 1')",
2411            )
2412            .await
2413            .unwrap_err();
2414        assert!(
2415            err.to_report_string()
2416                .contains("only accept shared CDC sources, not CDC table sources"),
2417            "{err:?}"
2418        );
2419
2420        frontend
2421            .run_sql("REVOKE SELECT ON SOURCE cdc_source FROM cdc_user")
2422            .await
2423            .unwrap();
2424        let err = frontend
2425            .run_sql_with_session(user_session, "ALTER TABLE cdc_table ADD COLUMN value INT")
2426            .await
2427            .unwrap_err();
2428        assert!(
2429            err.to_string()
2430                .contains("permission denied for source \"cdc_source\": \"SELECT\""),
2431            "{err:?}"
2432        );
2433    }
2434
2435    #[tokio::test]
2436    async fn test_create_table_handler() {
2437        let sql =
2438            "create table t (v1 smallint, v2 struct<v3 bigint, v4 float, v5 double>) append only;";
2439        let frontend = LocalFrontend::new(Default::default()).await;
2440        frontend.run_sql(sql).await.unwrap();
2441
2442        let session = frontend.session_ref();
2443        let catalog_reader = session.env().catalog_reader().read_guard();
2444        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
2445
2446        // Check table exists.
2447        let (table, _) = catalog_reader
2448            .get_created_table_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
2449            .unwrap();
2450        assert_eq!(table.name(), "t");
2451
2452        let columns = table
2453            .columns
2454            .iter()
2455            .map(|col| (col.name(), col.data_type().clone()))
2456            .collect::<HashMap<&str, DataType>>();
2457
2458        let expected_columns = maplit::hashmap! {
2459            ROW_ID_COLUMN_NAME => DataType::Serial,
2460            "v1" => DataType::Int16,
2461            "v2" => StructType::new(
2462                vec![("v3", DataType::Int64),("v4", DataType::Float64),("v5", DataType::Float64)],
2463            )
2464            .with_ids([3, 4, 5].map(ColumnId::new))
2465            .into(),
2466            RW_TIMESTAMP_COLUMN_NAME => DataType::Timestamptz,
2467        };
2468
2469        assert_eq!(columns, expected_columns, "{columns:#?}");
2470    }
2471
2472    #[tokio::test]
2473    async fn test_create_webhook_table_with_arbitrary_columns() {
2474        let frontend = LocalFrontend::new(Default::default()).await;
2475        frontend
2476            .run_sql("create schema ingest_schema;")
2477            .await
2478            .unwrap();
2479        frontend
2480            .run_sql(
2481                r#"
2482                create table ingest_schema.orders (
2483                    id int,
2484                    customer_name varchar,
2485                    amount double precision,
2486                    primary key (id)
2487                ) with (
2488                    connector = 'webhook'
2489                ) validate as secure_compare(
2490                    headers->>'x-rw-signature',
2491                    'sha256=' || encode(hmac('webhook-secret', payload, 'sha256'), 'hex')
2492                );
2493                "#,
2494            )
2495            .await
2496            .unwrap();
2497
2498        let session = frontend.session_ref();
2499        let catalog_reader = session.env().catalog_reader().read_guard();
2500        let (table, _) = catalog_reader
2501            .get_created_table_by_name(
2502                DEFAULT_DATABASE_NAME,
2503                SchemaPath::Name("ingest_schema"),
2504                "orders",
2505            )
2506            .unwrap();
2507
2508        assert!(table.webhook_info.is_some());
2509        assert_eq!(
2510            table
2511                .columns
2512                .iter()
2513                .filter(|column| column.can_dml())
2514                .count(),
2515            3
2516        );
2517    }
2518
2519    #[tokio::test]
2520    async fn test_create_webhook_table_uses_single_jsonb_column_name_in_validate() {
2521        let frontend = LocalFrontend::new(Default::default()).await;
2522        frontend
2523            .run_sql(
2524                r#"
2525                create table webhook_single_column (
2526                    body jsonb
2527                ) with (
2528                    connector = 'webhook'
2529                ) validate as secure_compare(
2530                    headers->>'x-rw-signature',
2531                    'sha256=' || encode(hmac('webhook-secret', body, 'sha256'), 'hex')
2532                );
2533                "#,
2534            )
2535            .await
2536            .unwrap();
2537    }
2538
2539    #[tokio::test]
2540    async fn test_create_webhook_table_with_generated_columns() {
2541        let frontend = LocalFrontend::new(Default::default()).await;
2542        let err = frontend
2543            .run_sql(
2544                r#"
2545                create table webhook_generated_columns (
2546                    id int,
2547                    amount double precision,
2548                    amount_with_fee double precision as amount + 1.0
2549                ) with (
2550                    connector = 'webhook'
2551                );
2552                "#,
2553            )
2554            .await
2555            .unwrap_err();
2556
2557        assert!(
2558            err.to_string()
2559                .contains("generated columns are not supported for webhook tables"),
2560            "{err:?}"
2561        );
2562    }
2563
2564    #[tokio::test]
2565    async fn test_create_webhook_table_with_default_value() {
2566        let frontend = LocalFrontend::new(Default::default()).await;
2567        let err = frontend
2568            .run_sql(
2569                r#"
2570                create table webhook_default_value (
2571                    id int default 42,
2572                    amount double precision
2573                ) with (
2574                    connector = 'webhook'
2575                );
2576                "#,
2577            )
2578            .await
2579            .unwrap_err();
2580
2581        assert!(
2582            err.to_string()
2583                .contains("default values are not supported for webhook tables"),
2584            "{err:?}"
2585        );
2586    }
2587
2588    #[tokio::test]
2589    async fn test_create_webhook_table_with_not_null_option() {
2590        let frontend = LocalFrontend::new(Default::default()).await;
2591        let err = frontend
2592            .run_sql(
2593                r#"
2594                create table webhook_not_null (
2595                    id int not null,
2596                    amount double precision
2597                ) with (
2598                    connector = 'webhook'
2599                );
2600                "#,
2601            )
2602            .await
2603            .unwrap_err();
2604
2605        assert!(
2606            err.to_string()
2607                .contains("only NULL column option is supported for webhook tables"),
2608            "{err:?}"
2609        );
2610    }
2611
2612    #[test]
2613    fn test_bind_primary_key() {
2614        // Note: Column ID 0 is reserved for row ID column.
2615
2616        for (sql, expected) in [
2617            ("create table t (v1 int, v2 int)", Ok(&[0] as &[_])),
2618            ("create table t (v1 int primary key, v2 int)", Ok(&[1])),
2619            ("create table t (v1 int, v2 int primary key)", Ok(&[2])),
2620            (
2621                "create table t (v1 int primary key, v2 int primary key)",
2622                Err("multiple primary keys are not allowed"),
2623            ),
2624            (
2625                "create table t (v1 int primary key primary key, v2 int)",
2626                Err("multiple primary keys are not allowed"),
2627            ),
2628            (
2629                "create table t (v1 int, v2 int, primary key (v1))",
2630                Ok(&[1]),
2631            ),
2632            (
2633                "create table t (v1 int, primary key (v2), v2 int)",
2634                Ok(&[2]),
2635            ),
2636            (
2637                "create table t (primary key (v2, v1), v1 int, v2 int)",
2638                Ok(&[2, 1]),
2639            ),
2640            (
2641                "create table t (v1 int, primary key (v1), v2 int, primary key (v1))",
2642                Err("multiple primary keys are not allowed"),
2643            ),
2644            (
2645                "create table t (v1 int primary key, primary key (v1), v2 int)",
2646                Err("multiple primary keys are not allowed"),
2647            ),
2648            (
2649                "create table t (v1 int, primary key (V3), v2 int)",
2650                Err("column \"v3\" named in key does not exist"),
2651            ),
2652        ] {
2653            let mut ast = risingwave_sqlparser::parser::Parser::parse_sql(sql).unwrap();
2654            let risingwave_sqlparser::ast::Statement::CreateTable {
2655                columns: column_defs,
2656                constraints,
2657                ..
2658            } = ast.remove(0)
2659            else {
2660                panic!("test case should be create table")
2661            };
2662            let actual: Result<_> = (|| {
2663                let mut columns = bind_sql_columns(&column_defs, false)?;
2664                let mut col_id_gen = ColumnIdGenerator::new_initial();
2665                for c in &mut columns {
2666                    col_id_gen.generate(c)?;
2667                }
2668
2669                let pk_names =
2670                    bind_sql_pk_names(&column_defs, bind_table_constraints(&constraints)?)?;
2671                let (_, pk_column_ids, _) =
2672                    bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
2673                Ok(pk_column_ids)
2674            })();
2675            match (expected, actual) {
2676                (Ok(expected), Ok(actual)) => assert_eq!(
2677                    expected.iter().copied().map(ColumnId::new).collect_vec(),
2678                    actual,
2679                    "sql: {sql}"
2680                ),
2681                (Ok(_), Err(actual)) => panic!("sql: {sql}\nunexpected error: {actual:?}"),
2682                (Err(_), Ok(actual)) => panic!("sql: {sql}\nexpects error but got: {actual:?}"),
2683                (Err(expected), Err(actual)) => assert!(
2684                    actual.to_string().contains(expected),
2685                    "sql: {sql}\nexpected: {expected:?}\nactual: {actual:?}"
2686                ),
2687            }
2688        }
2689    }
2690
2691    #[tokio::test]
2692    async fn test_duplicate_props_options() {
2693        let proto_file = create_proto_file(PROTO_FILE_DATA);
2694        let sql = format!(
2695            r#"CREATE TABLE t
2696    WITH (
2697        connector = 'kinesis',
2698        aws.region='user_test_topic',
2699        endpoint='172.10.1.1:9090,172.10.1.2:9090',
2700        aws.credentials.access_key_id = 'your_access_key_1',
2701        aws.credentials.secret_access_key = 'your_secret_key_1'
2702    )
2703    FORMAT PLAIN ENCODE PROTOBUF (
2704        message = '.test.TestRecord',
2705        aws.credentials.access_key_id = 'your_access_key_2',
2706        aws.credentials.secret_access_key = 'your_secret_key_2',
2707        schema.location = 'file://{}',
2708    )"#,
2709            proto_file.path().to_str().unwrap()
2710        );
2711        let frontend = LocalFrontend::new(Default::default()).await;
2712        frontend.run_sql(sql).await.unwrap();
2713
2714        let session = frontend.session_ref();
2715        let catalog_reader = session.env().catalog_reader().read_guard();
2716        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
2717
2718        // Check source exists.
2719        let (source, _) = catalog_reader
2720            .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
2721            .unwrap();
2722        assert_eq!(source.name, "t");
2723
2724        // AwsAuth params exist in options.
2725        assert_eq!(
2726            source
2727                .info
2728                .format_encode_options
2729                .get("aws.credentials.access_key_id")
2730                .unwrap(),
2731            "your_access_key_2"
2732        );
2733        assert_eq!(
2734            source
2735                .info
2736                .format_encode_options
2737                .get("aws.credentials.secret_access_key")
2738                .unwrap(),
2739            "your_secret_key_2"
2740        );
2741
2742        // AwsAuth params exist in props.
2743        assert_eq!(
2744            source
2745                .with_properties
2746                .get("aws.credentials.access_key_id")
2747                .unwrap(),
2748            "your_access_key_1"
2749        );
2750        assert_eq!(
2751            source
2752                .with_properties
2753                .get("aws.credentials.secret_access_key")
2754                .unwrap(),
2755            "your_secret_key_1"
2756        );
2757
2758        // Options are not merged into props.
2759        assert!(!source.with_properties.contains_key("schema.location"));
2760    }
2761}