1use std::collections::{BTreeMap, HashMap, HashSet};
16use std::rc::Rc;
17use std::sync::Arc;
18
19use anyhow::Context;
20use clap::ValueEnum;
21use either::Either;
22use fixedbitset::FixedBitSet;
23use iceberg::spec::FormatVersion;
24use itertools::Itertools;
25use percent_encoding::percent_decode_str;
26use pgwire::pg_response::{PgResponse, StatementType};
27use prost::Message as _;
28use risingwave_common::acl::AclMode;
29use risingwave_common::catalog::{
30 CdcKeyComparison, CdcTableDesc, ColumnCatalog, ColumnDesc, ConflictBehavior,
31 DEFAULT_SCHEMA_NAME, Engine, ICEBERG_SINK_PREFIX, ICEBERG_SOURCE_PREFIX,
32 RISINGWAVE_ICEBERG_ROW_ID, ROW_ID_COLUMN_NAME, TableId,
33};
34use risingwave_common::config::MetaBackend;
35use risingwave_common::global_jvm::Jvm;
36use risingwave_common::session_config::sink_decouple::SinkDecouple;
37use risingwave_common::types::DataType;
38use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
39use risingwave_common::util::value_encoding::DatumToProtoExt;
40use risingwave_common::{bail, bail_not_implemented};
41use risingwave_connector::source::cdc::external::ExternalCdcTableType;
42use risingwave_connector::source::cdc::{
43 CDC_COLUMN_EXCLUDE_LIST, CdcSourceExternalTableIdentifier, build_cdc_table_id,
44 parse_cdc_column_exclude_list,
45};
46use risingwave_connector::{
47 AUTO_SCHEMA_CHANGE_KEY, WithOptionsSecResolved, WithPropertiesExt, dispatch_cdc_source_type,
48};
49use risingwave_pb::catalog::connection::Info as ConnectionInfo;
50use risingwave_pb::catalog::connection_params::ConnectionType;
51use risingwave_pb::catalog::{PbSource, PbWebhookSourceInfo, WatermarkDesc};
52use risingwave_pb::ddl_service::{PbTableJobType, TableJobType};
53use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
54use risingwave_pb::plan_common::{
55 AdditionalColumn, ColumnDescVersion, DefaultColumnDesc, GeneratedColumnDesc,
56};
57use risingwave_pb::secret::PbSecretRef;
58use risingwave_pb::secret::secret_ref::PbRefAsType;
59use risingwave_pb::stream_plan::StreamFragmentGraph;
60use risingwave_sqlparser::ast::{
61 CdcTableInfo, ColumnDef, ColumnOption, CompatibleFormatEncode, ConnectionRefValue, CreateSink,
62 CreateSinkStatement, CreateSourceStatement, DataType as AstDataType, ExplainOptions, Format,
63 FormatEncodeOptions, Ident, ObjectName, OnConflict, SecretRefAsType, SourceWatermark,
64 Statement, TableConstraint, WebhookSourceInfo, WithProperties,
65};
66use risingwave_sqlparser::parser::IncludeOption;
67use thiserror_ext::AsReport;
68
69use super::RwPgResponse;
70use super::create_source::{CreateSourceType, SqlColumnStrategy, bind_columns_from_source};
71use crate::binder::{Clause, SecureCompareContext, WEBHOOK_PAYLOAD_FIELD_NAME, bind_data_type};
72use crate::catalog::root_catalog::SchemaPath;
73use crate::catalog::source_catalog::SourceCatalog;
74use crate::catalog::table_catalog::TableVersion;
75use crate::catalog::{ColumnId, DatabaseId, SchemaId, SourceId, check_column_name_not_reserved};
76use crate::error::{ErrorCode, Result, RwError, bail_bind_error};
77use crate::expr::{Expr, ExprImpl, ExprRewriter};
78use crate::handler::HandlerArgs;
79use crate::handler::cdc::{
80 bind_cdc_pk_comparisons_externally, bind_cdc_table_schema, bind_cdc_table_schema_externally,
81 derive_connect_properties_for_cdc_table, not_null_check_for_cdc_table,
82 reject_pk_filtered_by_debezium_column_filter, sanity_check_for_table_on_cdc_source,
83};
84use crate::handler::create_source::{
85 bind_connector_props, bind_create_source_or_table_with_connector, bind_source_watermark,
86 handle_addition_columns,
87};
88use crate::handler::privilege::ObjectCheckItem;
89use crate::handler::util::{
90 LongRunningNotificationAction, SourceSchemaCompatExt, execute_with_long_running_notification,
91};
92use crate::optimizer::plan_node::generic::{SourceNodeKind, build_cdc_scan_options_with_options};
93use crate::optimizer::plan_node::{
94 LogicalCdcScan, LogicalPlanRef, LogicalSource, StreamPlanRef as PlanRef,
95 ensure_sync_log_store_fragment_root,
96};
97use crate::optimizer::property::{Order, RequiredDist};
98use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRoot};
99use crate::session::SessionImpl;
100use crate::session::current::notice_to_user;
101use crate::stream_fragmenter::{GraphJobType, build_graph};
102use crate::utils::OverwriteOptions;
103use crate::{Binder, Explain, TableCatalog, WithOptions};
104
105mod col_id_gen;
106pub use col_id_gen::*;
107use risingwave_connector::sink::SinkParam;
108use risingwave_connector::sink::iceberg::{
109 ENABLE_COMPACTION, ENABLE_MANIFEST_REWRITE, IcebergConfig, IcebergSink,
110 is_iceberg_engine_option, parse_partition_by_exprs, validate_order_key_columns,
111};
112use risingwave_pb::ddl_service::create_iceberg_table_request::{PbSinkJobInfo, PbTableJobInfo};
113
114use crate::handler::create_sink::{SinkPlanContext, gen_sink_plan};
115
116fn ensure_column_options_supported(c: &ColumnDef) -> Result<()> {
117 for option_def in &c.options {
118 match option_def.option {
119 ColumnOption::GeneratedColumns(_) => {}
120 ColumnOption::DefaultValue(_) => {}
121 ColumnOption::DefaultValueInternal { .. } => {}
122 ColumnOption::Unique { is_primary: true } => {}
123 ColumnOption::Null => {}
124 ColumnOption::NotNull => {}
125 _ => bail_not_implemented!("column constraints \"{}\"", option_def),
126 }
127 }
128 Ok(())
129}
130
131pub fn bind_sql_columns(
135 column_defs: &[ColumnDef],
136 is_for_drop_table_connector: bool,
137) -> Result<Vec<ColumnCatalog>> {
138 let mut columns = Vec::with_capacity(column_defs.len());
139
140 for column in column_defs {
141 ensure_column_options_supported(column)?;
142 let ColumnDef {
146 name,
147 data_type,
148 collation,
149 options,
150 ..
151 } = column;
152
153 let data_type = data_type
154 .clone()
155 .ok_or_else(|| ErrorCode::InvalidInputSyntax("data type is not specified".into()))?;
156 if let Some(collation) = collation {
157 if !["C", "POSIX"].contains(&collation.real_value().as_str()) {
163 bail_not_implemented!(
164 "Collate collation other than `C` or `POSIX` is not implemented"
165 );
166 }
167
168 match data_type {
169 AstDataType::Text | AstDataType::Varchar | AstDataType::Char(_) => {}
170 _ => {
171 return Err(ErrorCode::NotSupported(
172 format!("{} is not a collatable data type", data_type),
173 "The only built-in collatable data types are `varchar`, please check your type".into(),
174 ).into());
175 }
176 }
177 }
178
179 if !is_for_drop_table_connector {
180 check_column_name_not_reserved(&name.real_value())?;
184 }
185
186 let nullable: bool = !options
187 .iter()
188 .any(|def| matches!(def.option, ColumnOption::NotNull));
189
190 columns.push(ColumnCatalog {
191 column_desc: ColumnDesc {
192 data_type: bind_data_type(&data_type)?,
193 column_id: ColumnId::placeholder(),
194 name: name.real_value(),
195 generated_or_default_column: None,
196 description: None,
197 additional_column: AdditionalColumn { column_type: None },
198 version: ColumnDescVersion::LATEST,
199 system_column: None,
200 nullable,
201 },
202 is_hidden: false,
203 });
204 }
205
206 Ok(columns)
207}
208
209fn check_generated_column_constraints(
210 column_name: &String,
211 column_id: ColumnId,
212 expr: &ExprImpl,
213 column_catalogs: &[ColumnCatalog],
214 generated_column_names: &[String],
215 pk_column_ids: &[ColumnId],
216) -> Result<()> {
217 let input_refs = expr.collect_input_refs(column_catalogs.len());
218 for idx in input_refs.ones() {
219 let referred_generated_column = &column_catalogs[idx].column_desc.name;
220 if generated_column_names
221 .iter()
222 .any(|c| c == referred_generated_column)
223 {
224 return Err(ErrorCode::BindError(format!(
225 "A generated column cannot reference another generated column. \
226 Generated column \"{}\" references generated column \"{}\"",
227 column_name, referred_generated_column
228 ))
229 .into());
230 }
231 }
232
233 if pk_column_ids.contains(&column_id) && expr.is_impure() {
234 return Err(ErrorCode::BindError(format!(
235 "Generated columns with impure expressions should not be part of the primary key. \
236 Here column \"{}\" is defined as part of the primary key.",
237 column_name
238 ))
239 .into());
240 }
241
242 Ok(())
243}
244
245pub fn bind_sql_column_constraints(
248 session: &SessionImpl,
249 table_name: String,
250 column_catalogs: &mut [ColumnCatalog],
251 columns: &[ColumnDef],
252 pk_column_ids: &[ColumnId],
253) -> Result<()> {
254 let generated_column_names = {
255 let mut names = vec![];
256 for column in columns {
257 for option_def in &column.options {
258 if let ColumnOption::GeneratedColumns(_) = option_def.option {
259 names.push(column.name.real_value());
260 break;
261 }
262 }
263 }
264 names
265 };
266
267 let mut binder = Binder::new_for_ddl(session);
268 binder.bind_columns_to_context(table_name, column_catalogs)?;
269
270 for column in columns {
271 let Some(idx) = column_catalogs
272 .iter()
273 .position(|c| c.name() == column.name.real_value())
274 else {
275 continue;
278 };
279
280 for option_def in &column.options {
281 match &option_def.option {
282 ColumnOption::GeneratedColumns(expr) => {
283 binder.set_clause(Some(Clause::GeneratedColumn));
284
285 let expr_impl = binder.bind_expr(expr).with_context(|| {
286 format!(
287 "failed to bind the expression in generated column \"{}\"",
288 column.name.real_value()
289 )
290 })?;
291
292 check_generated_column_constraints(
293 &column.name.real_value(),
294 column_catalogs[idx].column_id(),
295 &expr_impl,
296 column_catalogs,
297 &generated_column_names,
298 pk_column_ids,
299 )?;
300
301 column_catalogs[idx].column_desc.generated_or_default_column = Some(
302 GeneratedOrDefaultColumn::GeneratedColumn(GeneratedColumnDesc {
303 expr: Some(expr_impl.to_expr_proto()),
304 }),
305 );
306 binder.set_clause(None);
307 }
308 ColumnOption::DefaultValue(expr) => {
309 let expr_impl = binder
310 .bind_expr(expr)?
311 .cast_assign(column_catalogs[idx].data_type())?;
312
313 let rewritten_expr_impl = session
321 .pinned_snapshot()
322 .inline_now_proc_time()
323 .rewrite_expr(expr_impl.clone());
324
325 if let Some(snapshot_value) = rewritten_expr_impl.try_fold_const() {
326 let snapshot_value = snapshot_value?;
327
328 column_catalogs[idx].column_desc.generated_or_default_column =
329 Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
330 snapshot_value: Some(snapshot_value.to_protobuf()),
331 expr: Some(expr_impl.to_expr_proto()),
332 }));
334 } else {
335 return Err(ErrorCode::BindError(format!(
336 "Default expression used in column `{}` cannot be evaluated. \
337 Use generated columns instead if you mean to reference other columns.",
338 column.name
339 ))
340 .into());
341 }
342 }
343 ColumnOption::DefaultValueInternal { persisted, expr: _ } => {
344 if persisted.is_empty() {
349 bail_bind_error!(
350 "DEFAULT INTERNAL is only used for internal purposes, \
351 please specify a concrete default value"
352 );
353 }
354
355 let desc = DefaultColumnDesc::decode(&**persisted)
356 .expect("failed to decode persisted `DefaultColumnDesc`");
357
358 column_catalogs[idx].column_desc.generated_or_default_column =
359 Some(GeneratedOrDefaultColumn::DefaultColumn(desc));
360 }
361 _ => {}
362 }
363 }
364 }
365 Ok(())
366}
367
368pub fn bind_table_constraints(table_constraints: &[TableConstraint]) -> Result<Vec<String>> {
370 let mut pk_column_names = vec![];
371
372 for constraint in table_constraints {
373 match constraint {
374 TableConstraint::Unique {
375 name: _,
376 columns,
377 is_primary: true,
378 } => {
379 if !pk_column_names.is_empty() {
380 return Err(multiple_pk_definition_err());
381 }
382 pk_column_names = columns.iter().map(|c| c.real_value()).collect_vec();
383 }
384 _ => bail_not_implemented!("table constraint \"{}\"", constraint),
385 }
386 }
387 Ok(pk_column_names)
388}
389
390pub fn bind_sql_pk_names(
391 columns_defs: &[ColumnDef],
392 pk_names_from_table_constraints: Vec<String>,
393) -> Result<Vec<String>> {
394 let mut pk_column_names = pk_names_from_table_constraints;
395
396 for column in columns_defs {
397 for option_def in &column.options {
398 if let ColumnOption::Unique { is_primary: true } = option_def.option {
399 if !pk_column_names.is_empty() {
400 return Err(multiple_pk_definition_err());
401 }
402 pk_column_names.push(column.name.real_value());
403 };
404 }
405 }
406
407 Ok(pk_column_names)
408}
409
410fn multiple_pk_definition_err() -> RwError {
411 ErrorCode::BindError("multiple primary keys are not allowed".into()).into()
412}
413
414pub fn bind_pk_and_row_id_on_relation(
419 mut columns: Vec<ColumnCatalog>,
420 pk_names: Vec<String>,
421 must_need_pk: bool,
422) -> Result<(Vec<ColumnCatalog>, Vec<ColumnId>, Option<usize>)> {
423 for c in &columns {
424 assert!(c.column_id() != ColumnId::placeholder());
425 }
426
427 let name_to_id = columns
429 .iter()
430 .map(|c| (c.name(), c.column_id()))
431 .collect::<HashMap<_, _>>();
432
433 let mut pk_column_ids: Vec<_> = pk_names
434 .iter()
435 .map(|name| {
436 name_to_id.get(name.as_str()).copied().ok_or_else(|| {
437 ErrorCode::BindError(format!("column \"{name}\" named in key does not exist"))
438 })
439 })
440 .try_collect()?;
441
442 let need_row_id = pk_column_ids.is_empty() && must_need_pk;
444
445 let row_id_index = need_row_id.then(|| {
446 let column = ColumnCatalog::row_id_column();
447 let index = columns.len();
448 pk_column_ids = vec![column.column_id()];
449 columns.push(column);
450 index
451 });
452
453 if let Some(col) = columns.iter().map(|c| c.name()).duplicates().next() {
454 Err(ErrorCode::InvalidInputSyntax(format!(
455 "column \"{col}\" specified more than once"
456 )))?;
457 }
458
459 Ok((columns, pk_column_ids, row_id_index))
460}
461
462#[allow(clippy::too_many_arguments)]
465pub(crate) async fn gen_create_table_plan_with_source(
466 mut handler_args: HandlerArgs,
467 explain_options: ExplainOptions,
468 table_name: ObjectName,
469 column_defs: Vec<ColumnDef>,
470 wildcard_idx: Option<usize>,
471 constraints: Vec<TableConstraint>,
472 format_encode: FormatEncodeOptions,
473 source_watermarks: Vec<SourceWatermark>,
474 mut col_id_gen: ColumnIdGenerator,
475 include_column_options: IncludeOption,
476 props: CreateTableProps,
477 sql_column_strategy: SqlColumnStrategy,
478) -> Result<(PlanRef, Option<SourceCatalog>, TableCatalog)> {
479 if props.append_only
480 && format_encode.format != Format::Plain
481 && format_encode.format != Format::Native
482 {
483 return Err(ErrorCode::BindError(format!(
484 "Append only table does not support format {}.",
485 format_encode.format
486 ))
487 .into());
488 }
489
490 let session = &handler_args.session;
491 let (with_properties, ingest_mode) =
492 bind_connector_props(&handler_args, &format_encode, false)?;
493 if with_properties.is_shareable_cdc_connector() {
494 generated_columns_check_for_cdc_table(&column_defs)?;
495 not_null_check_for_cdc_table(&wildcard_idx, &column_defs)?;
496 } else if column_defs.iter().any(|col| {
497 col.options
498 .iter()
499 .any(|def| matches!(def.option, ColumnOption::NotNull))
500 }) {
501 notice_to_user(
503 "The table contains columns with NOT NULL constraints. Any rows from upstream violating the constraints will be ignored silently.",
504 );
505 }
506
507 let db_name: &str = &session.database();
508 let (schema_name, _) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
509 session.get_database_and_schema_id_for_create(schema_name.clone())?;
512
513 let (columns_from_resolve_source, source_info) = bind_columns_from_source(
515 session,
516 &format_encode,
517 Either::Left(&with_properties),
518 CreateSourceType::Table,
519 )
520 .await?;
521
522 let overwrite_options = OverwriteOptions::new(&mut handler_args)?;
523 let rate_limit = overwrite_options.source_rate_limit;
524 let source = bind_create_source_or_table_with_connector(
525 handler_args.clone(),
526 table_name,
527 format_encode,
528 with_properties,
529 &column_defs,
530 constraints,
531 wildcard_idx,
532 source_watermarks,
533 columns_from_resolve_source,
534 source_info,
535 include_column_options,
536 &mut col_id_gen,
537 CreateSourceType::Table,
538 rate_limit,
539 sql_column_strategy,
540 ingest_mode,
541 )
542 .await?;
543
544 let context = OptimizerContext::new(handler_args, explain_options)?;
545
546 let (plan, table) = gen_table_plan_with_source(
547 context.into(),
548 schema_name,
549 source.clone(),
550 col_id_gen.into_version(),
551 props,
552 )?;
553
554 Ok((plan, Some(source), table))
555}
556
557#[allow(clippy::too_many_arguments)]
560pub(crate) fn gen_create_table_plan(
561 context: OptimizerContext,
562 table_name: ObjectName,
563 column_defs: Vec<ColumnDef>,
564 constraints: Vec<TableConstraint>,
565 mut col_id_gen: ColumnIdGenerator,
566 source_watermarks: Vec<SourceWatermark>,
567 props: CreateTableProps,
568 is_for_replace_plan: bool,
569) -> Result<(PlanRef, TableCatalog)> {
570 let mut columns = bind_sql_columns(&column_defs, is_for_replace_plan)?;
571 for c in &mut columns {
572 col_id_gen.generate(c)?;
573 }
574
575 let (_, secret_refs, connection_refs) = context.with_options().clone().into_parts();
576 if !secret_refs.is_empty() || !connection_refs.is_empty() {
577 return Err(crate::error::ErrorCode::InvalidParameterValue("Secret reference and Connection reference are not allowed in options when creating table without external source".to_owned()).into());
578 }
579
580 gen_create_table_plan_without_source(
581 context,
582 table_name,
583 columns,
584 column_defs,
585 constraints,
586 source_watermarks,
587 col_id_gen.into_version(),
588 props,
589 )
590}
591
592#[allow(clippy::too_many_arguments)]
593pub(crate) fn gen_create_table_plan_without_source(
594 context: OptimizerContext,
595 table_name: ObjectName,
596 columns: Vec<ColumnCatalog>,
597 column_defs: Vec<ColumnDef>,
598 constraints: Vec<TableConstraint>,
599 source_watermarks: Vec<SourceWatermark>,
600 version: TableVersion,
601 props: CreateTableProps,
602) -> Result<(PlanRef, TableCatalog)> {
603 let pk_names = bind_sql_pk_names(&column_defs, bind_table_constraints(&constraints)?)?;
605 let (mut columns, pk_column_ids, row_id_index) =
606 bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
607
608 let watermark_descs = bind_source_watermark(
609 context.session_ctx(),
610 table_name.real_value(),
611 source_watermarks,
612 &columns,
613 )?;
614
615 bind_sql_column_constraints(
616 context.session_ctx(),
617 table_name.real_value(),
618 &mut columns,
619 &column_defs,
620 &pk_column_ids,
621 )?;
622 let session = context.session_ctx().clone();
623
624 let db_name = &session.database();
625 let (schema_name, table_name) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
626
627 let info = CreateTableInfo {
628 columns,
629 pk_column_ids,
630 row_id_index,
631 watermark_descs,
632 source_catalog: None,
633 version,
634 };
635
636 gen_table_plan_inner(context.into(), schema_name, table_name, info, props)
637}
638
639fn gen_table_plan_with_source(
640 context: OptimizerContextRef,
641 schema_name: Option<String>,
642 source_catalog: SourceCatalog,
643 version: TableVersion,
644 props: CreateTableProps,
645) -> Result<(PlanRef, TableCatalog)> {
646 let table_name = source_catalog.name.clone();
647
648 let info = CreateTableInfo {
649 columns: source_catalog.columns.clone(),
650 pk_column_ids: source_catalog.pk_col_ids.clone(),
651 row_id_index: source_catalog.row_id_index,
652 watermark_descs: source_catalog.watermark_descs.clone(),
653 source_catalog: Some(source_catalog),
654 version,
655 };
656
657 gen_table_plan_inner(context, schema_name, table_name, info, props)
658}
659
660#[derive(Clone, Copy)]
662pub enum EitherOnConflict {
663 Ast(Option<OnConflict>),
664 Resolved(ConflictBehavior),
665}
666
667impl From<Option<OnConflict>> for EitherOnConflict {
668 fn from(v: Option<OnConflict>) -> Self {
669 Self::Ast(v)
670 }
671}
672
673impl From<ConflictBehavior> for EitherOnConflict {
674 fn from(v: ConflictBehavior) -> Self {
675 Self::Resolved(v)
676 }
677}
678
679impl EitherOnConflict {
680 pub fn to_behavior(self, append_only: bool, row_id_as_pk: bool) -> Result<ConflictBehavior> {
682 let conflict_behavior = match self {
683 EitherOnConflict::Ast(on_conflict) => {
684 if append_only {
685 if row_id_as_pk {
686 ConflictBehavior::NoCheck
688 } else {
689 if let Some(on_conflict) = on_conflict
691 && on_conflict != OnConflict::Nothing
692 {
693 return Err(ErrorCode::InvalidInputSyntax(
694 "When PRIMARY KEY constraint applied to an APPEND ONLY table, \
695 the ON CONFLICT behavior must be DO NOTHING."
696 .to_owned(),
697 )
698 .into());
699 }
700 ConflictBehavior::IgnoreConflict
701 }
702 } else {
703 match on_conflict.unwrap_or(OnConflict::UpdateFull) {
705 OnConflict::UpdateFull => ConflictBehavior::Overwrite,
706 OnConflict::Nothing => ConflictBehavior::IgnoreConflict,
707 OnConflict::UpdateIfNotNull => ConflictBehavior::DoUpdateIfNotNull,
708 }
709 }
710 }
711 EitherOnConflict::Resolved(b) => b,
712 };
713
714 Ok(conflict_behavior)
715 }
716}
717
718pub struct CreateTableInfo {
723 pub columns: Vec<ColumnCatalog>,
724 pub pk_column_ids: Vec<ColumnId>,
725 pub row_id_index: Option<usize>,
726 pub watermark_descs: Vec<WatermarkDesc>,
727 pub source_catalog: Option<SourceCatalog>,
728 pub version: TableVersion,
729}
730
731pub struct CreateTableProps {
736 pub definition: String,
737 pub append_only: bool,
738 pub on_conflict: EitherOnConflict,
739 pub with_version_columns: Vec<String>,
740 pub webhook_info: Option<PbWebhookSourceInfo>,
741 pub engine: Engine,
742}
743
744#[allow(clippy::too_many_arguments)]
745fn gen_table_plan_inner(
746 context: OptimizerContextRef,
747 schema_name: Option<String>,
748 table_name: String,
749 info: CreateTableInfo,
750 props: CreateTableProps,
751) -> Result<(PlanRef, TableCatalog)> {
752 let CreateTableInfo {
753 ref columns,
754 row_id_index,
755 ref watermark_descs,
756 ref source_catalog,
757 ..
758 } = info;
759 let CreateTableProps { append_only, .. } = props;
760
761 let (database_id, schema_id) = context
762 .session_ctx()
763 .get_database_and_schema_id_for_create(schema_name)?;
764
765 let session = context.session_ctx().clone();
766 let retention_seconds = context.with_options().retention_seconds();
767
768 let source_node: LogicalPlanRef = LogicalSource::new(
769 source_catalog.clone().map(Rc::new),
770 columns.clone(),
771 row_id_index,
772 SourceNodeKind::CreateTable,
773 context.clone(),
774 None,
775 )?
776 .into();
777
778 let required_cols = FixedBitSet::with_capacity(columns.len());
779 let plan_root = PlanRoot::new_with_logical_plan(
780 source_node,
781 RequiredDist::Any,
782 Order::any(),
783 required_cols,
784 vec![],
785 );
786
787 let has_non_ttl_watermark = watermark_descs.iter().any(|d| !d.with_ttl);
788
789 if !append_only && has_non_ttl_watermark {
790 return Err(ErrorCode::NotSupported(
791 "Defining watermarks on table requires the table to be append only.".to_owned(),
792 "Use the key words `APPEND ONLY`".to_owned(),
793 )
794 .into());
795 }
796
797 if !append_only && retention_seconds.is_some() {
798 if session
799 .config()
800 .unsafe_enable_storage_retention_for_non_append_only_tables()
801 {
802 tracing::warn!(
803 "Storage retention is enabled for non-append-only table {}. This may lead to stream inconsistency.",
804 table_name
805 );
806 const NOTICE: &str = "Storage retention is enabled for non-append-only table. \
807 This may lead to stream inconsistency and unrecoverable \
808 node failure if there is any row INSERT/UPDATE/DELETE operation \
809 corresponding to the TTLed primary keys";
810 session.notice_to_user(NOTICE);
811 } else {
812 return Err(ErrorCode::NotSupported(
813 "Defining retention seconds on table requires the table to be append only."
814 .to_owned(),
815 "Use the key words `APPEND ONLY`".to_owned(),
816 )
817 .into());
818 }
819 }
820
821 let materialize =
822 plan_root.gen_table_plan(context, table_name, database_id, schema_id, info, props)?;
823
824 let mut table = materialize.table().clone();
825 table.owner = session.user_id();
826
827 Ok((
828 ensure_sync_log_store_fragment_root(materialize.into()),
829 table,
830 ))
831}
832
833#[allow(clippy::too_many_arguments)]
837pub(crate) fn gen_create_table_plan_for_cdc_table(
838 context: OptimizerContextRef,
839 source: Arc<SourceCatalog>,
840 external_table_name: String,
841 column_defs: Vec<ColumnDef>,
842 source_watermarks: Vec<SourceWatermark>,
843 mut columns: Vec<ColumnCatalog>,
844 pk_names: Vec<String>,
845 pk_comparisons: Vec<CdcKeyComparison>,
846 cdc_with_options: WithOptionsSecResolved,
847 mut col_id_gen: ColumnIdGenerator,
848 on_conflict: Option<OnConflict>,
849 with_version_columns: Vec<String>,
850 include_column_options: IncludeOption,
851 table_name: ObjectName,
852 resolved_table_name: String, database_id: DatabaseId,
854 schema_id: SchemaId,
855 table_id: TableId,
856 engine: Engine,
857) -> Result<(PlanRef, TableCatalog)> {
858 let session = context.session_ctx().clone();
859
860 handle_addition_columns(
862 None,
863 &cdc_with_options,
864 include_column_options,
865 &mut columns,
866 true,
867 )?;
868
869 for c in &mut columns {
870 col_id_gen.generate(c)?;
871 }
872
873 let (mut columns, pk_column_ids, _row_id_index) =
874 bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
875
876 let watermark_descs = bind_source_watermark(
877 context.session_ctx(),
878 table_name.real_value(),
879 source_watermarks,
880 &columns,
881 )?;
882
883 bind_sql_column_constraints(
885 context.session_ctx(),
886 table_name.real_value(),
887 &mut columns,
888 &column_defs,
889 &pk_column_ids,
890 )?;
891
892 let definition = context.normalized_sql().to_owned();
893
894 let pk_column_indices = {
895 let mut id_to_idx = HashMap::new();
896 columns.iter().enumerate().for_each(|(idx, c)| {
897 id_to_idx.insert(c.column_id(), idx);
898 });
899 pk_column_ids
901 .iter()
902 .map(|c| id_to_idx.get(c).copied().unwrap())
903 .collect_vec()
904 };
905 let table_pk = pk_column_indices
906 .iter()
907 .map(|idx| ColumnOrder::new(*idx, OrderType::ascending()))
908 .collect();
909
910 let (options, secret_refs) = cdc_with_options.into_parts();
911
912 let non_generated_column_descs = columns
913 .iter()
914 .filter(|&c| !c.is_generated())
915 .map(|c| c.column_desc.clone())
916 .collect_vec();
917 let non_generated_column_num = non_generated_column_descs.len();
918 let cdc_table_type = ExternalCdcTableType::from_properties(&options);
919 let canonical_name = dispatch_cdc_source_type!(
920 options.cdc_source_type(),
921 T,
922 T::canonical_name(&options),
923 unreachable!("CDC table without a CDC connector")
924 );
925 let cdc_table_desc = CdcTableDesc {
926 table_id,
927 source_id: source.id, external_table_name: external_table_name.clone(),
929 pk: table_pk,
930 pk_comparisons,
931 columns: non_generated_column_descs,
932 stream_key: pk_column_indices,
933 connect_properties: options,
934 secret_refs,
935 };
936
937 tracing::debug!(?cdc_table_desc, "create cdc table");
938 let options = build_cdc_scan_options_with_options(
939 context.with_options(),
940 cdc_table_desc.connect_properties.cdc_source_type(),
941 )?;
942
943 let logical_scan = LogicalCdcScan::create(
944 external_table_name,
945 Rc::new(cdc_table_desc),
946 context.clone(),
947 options,
948 );
949
950 let scan_node: LogicalPlanRef = logical_scan.into();
951 let required_cols = FixedBitSet::with_capacity(non_generated_column_num);
952 let plan_root = PlanRoot::new_with_logical_plan(
953 scan_node,
954 RequiredDist::Any,
955 Order::any(),
956 required_cols,
957 vec![],
958 );
959
960 let cdc_table_id = build_cdc_table_id(source.id, &canonical_name);
961 let materialize = plan_root.gen_table_plan(
962 context,
963 resolved_table_name,
964 database_id,
965 schema_id,
966 CreateTableInfo {
967 columns,
968 pk_column_ids,
969 row_id_index: None,
970 watermark_descs,
971 source_catalog: Some((*source).clone()),
972 version: col_id_gen.into_version(),
973 },
974 CreateTableProps {
975 definition,
976 append_only: false,
977 on_conflict: on_conflict.into(),
978 with_version_columns,
979 webhook_info: None,
980 engine,
981 },
982 )?;
983
984 let mut table = materialize.table().clone();
985 table.owner = session.user_id();
986 table.cdc_table_id = Some(cdc_table_id);
987 table.cdc_table_type = Some(cdc_table_type);
988 Ok((
989 ensure_sync_log_store_fragment_root(materialize.into()),
990 table,
991 ))
992}
993
994#[allow(clippy::too_many_arguments)]
995pub(super) async fn handle_create_table_plan(
996 handler_args: HandlerArgs,
997 explain_options: ExplainOptions,
998 format_encode: Option<FormatEncodeOptions>,
999 cdc_table_info: Option<CdcTableInfo>,
1000 table_name: &ObjectName,
1001 column_defs: Vec<ColumnDef>,
1002 wildcard_idx: Option<usize>,
1003 constraints: Vec<TableConstraint>,
1004 source_watermarks: Vec<SourceWatermark>,
1005 append_only: bool,
1006 on_conflict: Option<OnConflict>,
1007 with_version_columns: Vec<String>,
1008 include_column_options: IncludeOption,
1009 webhook_info: Option<WebhookSourceInfo>,
1010 engine: Engine,
1011) -> Result<(
1012 PlanRef,
1013 Option<SourceCatalog>,
1014 TableCatalog,
1015 TableJobType,
1016 Option<SourceId>,
1017)> {
1018 let col_id_gen = ColumnIdGenerator::new_initial();
1019 let format_encode = check_create_table_with_source(
1020 &handler_args.with_options,
1021 format_encode,
1022 &include_column_options,
1023 &cdc_table_info,
1024 )?;
1025 let webhook_info = webhook_info
1026 .map(|info| bind_webhook_info(&handler_args.session, &column_defs, info))
1027 .transpose()?;
1028
1029 let props = CreateTableProps {
1030 definition: handler_args.normalized_sql.clone(),
1031 append_only,
1032 on_conflict: on_conflict.into(),
1033 with_version_columns: with_version_columns.clone(),
1034 webhook_info,
1035 engine,
1036 };
1037
1038 let ((plan, source, table), job_type, shared_shource_id) = match (
1039 format_encode,
1040 cdc_table_info.as_ref(),
1041 ) {
1042 (Some(format_encode), None) => (
1043 gen_create_table_plan_with_source(
1044 handler_args,
1045 explain_options,
1046 table_name.clone(),
1047 column_defs,
1048 wildcard_idx,
1049 constraints,
1050 format_encode,
1051 source_watermarks,
1052 col_id_gen,
1053 include_column_options,
1054 props,
1055 SqlColumnStrategy::FollowChecked,
1056 )
1057 .await?,
1058 TableJobType::General,
1059 None,
1060 ),
1061 (None, None) => {
1062 let context = OptimizerContext::new(handler_args, explain_options)?;
1063 let (plan, table) = gen_create_table_plan(
1064 context,
1065 table_name.clone(),
1066 column_defs,
1067 constraints,
1068 col_id_gen,
1069 source_watermarks,
1070 props,
1071 false,
1072 )?;
1073
1074 ((plan, None, table), TableJobType::General, None)
1075 }
1076
1077 (None, Some(cdc_table)) => {
1078 sanity_check_for_table_on_cdc_source(
1079 append_only,
1080 &column_defs,
1081 &wildcard_idx,
1082 &constraints,
1083 &source_watermarks,
1084 )?;
1085
1086 generated_columns_check_for_cdc_table(&column_defs)?;
1087 not_null_check_for_cdc_table(&wildcard_idx, &column_defs)?;
1088
1089 let session = &handler_args.session;
1090 let db_name = &session.database();
1091 let user_name = &session.user_name();
1092 let search_path = session.config().search_path();
1093 let (schema_name, resolved_table_name) =
1094 Binder::resolve_schema_qualified_name(db_name, table_name)?;
1095 let (database_id, schema_id) =
1096 session.get_database_and_schema_id_for_create(schema_name.clone())?;
1097
1098 let (source_schema, source_name) =
1100 Binder::resolve_schema_qualified_name(db_name, &cdc_table.source_name)?;
1101
1102 let source = {
1103 let catalog_reader = session.env().catalog_reader().read_guard();
1104 let schema_path =
1105 SchemaPath::new(source_schema.as_deref(), &search_path, user_name);
1106
1107 let (source, _) = catalog_reader.get_source_by_name(
1108 db_name,
1109 schema_path,
1110 source_name.as_str(),
1111 )?;
1112 source.clone()
1113 };
1114 check_cdc_source_select_privilege(session, &source)?;
1115 let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1116 &source.with_properties,
1117 cdc_table.external_table_name.clone(),
1118 )?;
1119
1120 let excluded_columns = match handler_args.with_options.get(CDC_COLUMN_EXCLUDE_LIST) {
1121 Some(list) => {
1122 let columns = parse_cdc_column_exclude_list(list);
1123 if columns.is_empty() {
1124 return Err(ErrorCode::InvalidInputSyntax(format!(
1125 "`{CDC_COLUMN_EXCLUDE_LIST}` must list at least one column"
1126 ))
1127 .into());
1128 }
1129 if wildcard_idx.is_none() {
1130 return Err(ErrorCode::InvalidInputSyntax(format!(
1131 "`{CDC_COLUMN_EXCLUDE_LIST}` requires the `(*)` wildcard in the column list"
1132 ))
1133 .into());
1134 }
1135 columns
1136 }
1137 None => HashSet::new(),
1138 };
1139
1140 let (columns, pk_names, pk_comparisons) = match wildcard_idx {
1141 Some(_) => {
1142 bind_cdc_table_schema_externally(cdc_with_options.clone(), &excluded_columns)
1143 .await?
1144 }
1145 None => {
1146 for column_def in &column_defs {
1147 for option_def in &column_def.options {
1148 if let ColumnOption::DefaultValue(_)
1149 | ColumnOption::DefaultValueInternal { .. } = option_def.option
1150 {
1151 return Err(ErrorCode::NotSupported(
1152 "Default value for columns defined on the table created from a CDC source".into(),
1153 "Remove the default value expression in the column definitions".into(),
1154 )
1155 .into());
1156 }
1157 }
1158 }
1159
1160 let (columns, pk_names) =
1161 bind_cdc_table_schema(&column_defs, &constraints, false)?;
1162 let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1163 cdc_with_options.clone(),
1164 &pk_names,
1165 ))
1166 .await?;
1167
1168 (columns, pk_names, pk_comparisons)
1169 }
1170 };
1171
1172 reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
1177
1178 let context: OptimizerContextRef =
1179 OptimizerContext::new(handler_args, explain_options)?.into();
1180 let shared_source_id = source.id;
1181 let (plan, table) = gen_create_table_plan_for_cdc_table(
1182 context,
1183 source,
1184 external_table_name,
1185 column_defs,
1186 source_watermarks,
1187 columns,
1188 pk_names,
1189 pk_comparisons,
1190 cdc_with_options,
1191 col_id_gen,
1192 on_conflict,
1193 with_version_columns,
1194 include_column_options,
1195 table_name.clone(),
1196 resolved_table_name,
1197 database_id,
1198 schema_id,
1199 TableId::placeholder(),
1200 engine,
1201 )?;
1202
1203 (
1204 (plan, None, table),
1205 TableJobType::SharedCdcSource,
1206 Some(shared_source_id),
1207 )
1208 }
1209 (Some(_), Some(_)) => {
1210 return Err(ErrorCode::NotSupported(
1211 "Data format and encoding format doesn't apply to table created from a CDC source"
1212 .into(),
1213 "Remove the FORMAT and ENCODE specification".into(),
1214 )
1215 .into());
1216 }
1217 };
1218 Ok((plan, source, table, job_type, shared_shource_id))
1219}
1220
1221fn generated_columns_check_for_cdc_table(columns: &Vec<ColumnDef>) -> Result<()> {
1223 let mut found_generated_column = false;
1224 for column in columns {
1225 let mut is_generated = false;
1226
1227 for option_def in &column.options {
1228 if let ColumnOption::GeneratedColumns(_) = option_def.option {
1229 is_generated = true;
1230 break;
1231 }
1232 }
1233
1234 if is_generated {
1235 found_generated_column = true;
1236 } else if found_generated_column {
1237 return Err(ErrorCode::NotSupported(
1238 "Non-generated column found after a generated column.".into(),
1239 "Ensure that all generated columns appear at the end of the cdc table definition."
1240 .into(),
1241 )
1242 .into());
1243 }
1244 }
1245 Ok(())
1246}
1247
1248#[allow(clippy::too_many_arguments)]
1249pub async fn handle_create_table(
1250 handler_args: HandlerArgs,
1251 table_name: ObjectName,
1252 column_defs: Vec<ColumnDef>,
1253 wildcard_idx: Option<usize>,
1254 constraints: Vec<TableConstraint>,
1255 if_not_exists: bool,
1256 format_encode: Option<FormatEncodeOptions>,
1257 source_watermarks: Vec<SourceWatermark>,
1258 append_only: bool,
1259 on_conflict: Option<OnConflict>,
1260 with_version_columns: Vec<String>,
1261 cdc_table_info: Option<CdcTableInfo>,
1262 include_column_options: IncludeOption,
1263 webhook_info: Option<WebhookSourceInfo>,
1264 ast_engine: risingwave_sqlparser::ast::Engine,
1265) -> Result<RwPgResponse> {
1266 let session = handler_args.session.clone();
1267
1268 if append_only {
1269 session.notice_to_user("APPEND ONLY TABLE is currently an experimental feature.");
1270 }
1271
1272 session.check_cluster_limits().await?;
1273
1274 let engine = match ast_engine {
1275 risingwave_sqlparser::ast::Engine::Hummock => Engine::Hummock,
1276 risingwave_sqlparser::ast::Engine::Iceberg => Engine::Iceberg,
1277 };
1278
1279 if let Either::Right(resp) = session.check_relation_name_duplicated(
1280 table_name.clone(),
1281 StatementType::CREATE_TABLE,
1282 if_not_exists,
1283 )? {
1284 return Ok(resp);
1285 }
1286
1287 let (graph, source, hummock_table, job_type, shared_source_id) = {
1288 let (plan, source, table, job_type, shared_source_id) = Box::pin(handle_create_table_plan(
1289 handler_args.clone(),
1290 ExplainOptions::default(),
1291 format_encode,
1292 cdc_table_info,
1293 &table_name,
1294 column_defs.clone(),
1295 wildcard_idx,
1296 constraints.clone(),
1297 source_watermarks,
1298 append_only,
1299 on_conflict,
1300 with_version_columns,
1301 include_column_options,
1302 webhook_info,
1303 engine,
1304 ))
1305 .await?;
1306 tracing::trace!("table_plan: {:?}", plan.explain_to_string());
1307
1308 let graph = build_graph(plan, Some(GraphJobType::Table))?;
1309
1310 (graph, source, table, job_type, shared_source_id)
1311 };
1312
1313 tracing::trace!(
1314 "name={}, graph=\n{}",
1315 table_name,
1316 serde_json::to_string_pretty(&graph).unwrap()
1317 );
1318
1319 let dependencies = shared_source_id
1320 .map(|id| HashSet::from([id.as_object_id()]))
1321 .unwrap_or_default();
1322
1323 match engine {
1325 Engine::Hummock => {
1326 let catalog_writer = session.catalog_writer()?;
1327 let action = match job_type {
1328 TableJobType::SharedCdcSource => LongRunningNotificationAction::MonitorBackfillJob,
1329 _ => LongRunningNotificationAction::DiagnoseBarrierLatency,
1330 };
1331 execute_with_long_running_notification(
1332 catalog_writer.create_table(
1333 source.map(|s| s.to_prost()),
1334 hummock_table.to_prost(),
1335 graph,
1336 job_type,
1337 if_not_exists,
1338 dependencies,
1339 ),
1340 &session,
1341 "CREATE TABLE",
1342 action,
1343 )
1344 .await?;
1345 }
1346 Engine::Iceberg => {
1347 let hummock_table_name = hummock_table.name.clone();
1348 session.create_staging_table(hummock_table.clone());
1349 let res = Box::pin(create_iceberg_engine_table(
1350 session.clone(),
1351 handler_args,
1352 source.map(|s| s.to_prost()),
1353 hummock_table,
1354 graph,
1355 table_name,
1356 job_type,
1357 if_not_exists,
1358 ))
1359 .await;
1360 session.drop_staging_table(&hummock_table_name);
1361 res?
1362 }
1363 }
1364
1365 Ok(PgResponse::empty_result(StatementType::CREATE_TABLE))
1366}
1367
1368fn build_iceberg_engine_sink_options(
1369 mut sink_options: BTreeMap<String, String>,
1370 user_options: &WithOptions,
1371 table: &TableCatalog,
1372 primary_key: &[String],
1373) -> Result<BTreeMap<String, String>> {
1374 sink_options.extend(
1375 user_options
1376 .iter()
1377 .filter(|(key, _)| is_iceberg_engine_option(key))
1378 .map(|(key, value)| (key.clone(), value.clone())),
1379 );
1380
1381 sink_options
1382 .entry(ENABLE_COMPACTION.to_owned())
1383 .or_insert_with(|| "true".to_owned());
1384 sink_options.insert(
1385 "type".to_owned(),
1386 if table.append_only {
1387 "append-only"
1388 } else {
1389 "upsert"
1390 }
1391 .to_owned(),
1392 );
1393
1394 if !table.append_only {
1397 sink_options.insert("primary_key".to_owned(), primary_key.join(","));
1398 }
1399
1400 sink_options.insert("create_table_if_not_exists".to_owned(), "true".to_owned());
1401 sink_options.insert("is_exactly_once".to_owned(), "true".to_owned());
1402
1403 let config = IcebergConfig::from_btreemap(sink_options.clone())?;
1404
1405 if !table.append_only && !config.enable_pk_index {
1408 for pk in table.pk() {
1409 let column = &table.columns()[pk.column_index];
1410 let data_type = column.data_type();
1411 if data_type.is_composite()
1412 || matches!(data_type, DataType::Float32 | DataType::Float64)
1413 {
1414 return Err(ErrorCode::NotSupported(
1415 format!(
1416 "Iceberg engine table primary key column \"{}\" has unsupported type {} for equality deletes",
1417 column.name(), data_type
1418 ),
1419 "Use non-floating-point scalar primary key columns.".to_owned(),
1420 )
1421 .into());
1422 }
1423 }
1424 }
1425
1426 if config.table_format_version() < FormatVersion::V3 {
1429 sink_options
1430 .entry(ENABLE_MANIFEST_REWRITE.to_owned())
1431 .or_insert_with(|| "true".to_owned());
1432 }
1433
1434 if let Some(partition_by) = &config.partition_by {
1435 let mut partition_columns = vec![];
1436 for (column, _) in parse_partition_by_exprs(partition_by.clone())? {
1437 table
1438 .columns()
1439 .iter()
1440 .find(|col| col.name().eq_ignore_ascii_case(&column))
1441 .ok_or_else(|| {
1442 ErrorCode::InvalidInputSyntax(format!(
1443 "Partition source column does not exist in schema: {}",
1444 column
1445 ))
1446 })?;
1447
1448 partition_columns.push(column);
1449 }
1450
1451 ensure_partition_columns_are_prefix_of_primary_key(&partition_columns, primary_key)
1452 .map_err(|_| {
1453 ErrorCode::InvalidInputSyntax(
1454 "The partition columns should be the prefix of the primary key".to_owned(),
1455 )
1456 })?;
1457 }
1458
1459 if let Some(order_key) = &config.order_key {
1460 validate_order_key_columns(order_key, table.columns().iter().map(|col| col.name()))
1461 .map_err(|err| ErrorCode::InvalidInputSyntax(err.to_report_string()))?;
1462 }
1463
1464 if config.enable_pk_index {
1465 sink_options.remove("primary_key");
1466 } else {
1467 sink_options.insert(AUTO_SCHEMA_CHANGE_KEY.to_owned(), "true".to_owned());
1468 }
1469
1470 Ok(sink_options)
1471}
1472
1473#[allow(clippy::too_many_arguments)]
1482pub async fn create_iceberg_engine_table(
1483 session: Arc<SessionImpl>,
1484 handler_args: HandlerArgs,
1485 mut source: Option<PbSource>,
1486 table: TableCatalog,
1487 graph: StreamFragmentGraph,
1488 table_name: ObjectName,
1489 job_type: PbTableJobType,
1490 if_not_exists: bool,
1491) -> Result<()> {
1492 let rw_db_name = session
1493 .env()
1494 .catalog_reader()
1495 .read_guard()
1496 .get_database_by_id(table.database_id)?
1497 .name()
1498 .to_owned();
1499 let rw_schema_name = session
1500 .env()
1501 .catalog_reader()
1502 .read_guard()
1503 .get_schema_by_id(table.database_id, table.schema_id)?
1504 .name()
1505 .clone();
1506 let iceberg_catalog_name = rw_db_name.clone();
1507 let iceberg_database_name = rw_schema_name.clone();
1508 let iceberg_table_name = table_name.0.last().unwrap().real_value();
1509
1510 let iceberg_engine_connection: String = session.config().iceberg_engine_connection();
1511 let sink_decouple = session.config().sink_decouple();
1512 if matches!(sink_decouple, SinkDecouple::Disable) {
1513 bail!(
1514 "Iceberg engine table only supports with sink decouple, try `set sink_decouple = true` to resolve it"
1515 );
1516 }
1517
1518 let mut connection_ref = BTreeMap::new();
1519 let with_common = if iceberg_engine_connection.is_empty() {
1520 bail!("to use iceberg engine table, the variable `iceberg_engine_connection` must be set.");
1521 } else {
1522 let parts: Vec<&str> = iceberg_engine_connection.split('.').collect();
1523 assert_eq!(parts.len(), 2);
1524 let connection_catalog =
1525 session.get_connection_by_name(Some(parts[0].to_owned()), parts[1])?;
1526 if let ConnectionInfo::ConnectionParams(params) = &connection_catalog.info {
1527 if params.connection_type == ConnectionType::Iceberg as i32 {
1528 connection_ref.insert(
1530 "connection".to_owned(),
1531 ConnectionRefValue {
1532 connection_name: ObjectName::from(vec![
1533 Ident::from(parts[0]),
1534 Ident::from(parts[1]),
1535 ]),
1536 },
1537 );
1538
1539 let mut with_common = BTreeMap::new();
1540 with_common.insert("connector".to_owned(), "iceberg".to_owned());
1541 with_common.insert("database.name".to_owned(), iceberg_database_name);
1542 with_common.insert("table.name".to_owned(), iceberg_table_name);
1543
1544 let hosted_catalog = params
1545 .properties
1546 .get("hosted_catalog")
1547 .map(|s| s.eq_ignore_ascii_case("true"))
1548 .unwrap_or(false);
1549 if hosted_catalog {
1550 let meta_client = session.env().meta_client();
1551 let meta_store_endpoint = meta_client.get_meta_store_endpoint().await?;
1552
1553 let meta_store_endpoint =
1554 url::Url::parse(&meta_store_endpoint).map_err(|_| {
1555 ErrorCode::InternalError(
1556 "failed to parse the meta store endpoint".to_owned(),
1557 )
1558 })?;
1559 let meta_store_backend = meta_store_endpoint.scheme().to_owned();
1560 let meta_store_user = meta_store_endpoint.username().to_owned();
1561 let meta_store_password = match meta_store_endpoint.password() {
1562 Some(password) => percent_decode_str(password)
1563 .decode_utf8()
1564 .map_err(|_| {
1565 ErrorCode::InternalError(
1566 "failed to parse password from meta store endpoint".to_owned(),
1567 )
1568 })?
1569 .into_owned(),
1570 None => "".to_owned(),
1571 };
1572 let meta_store_host = meta_store_endpoint
1573 .host_str()
1574 .ok_or_else(|| {
1575 ErrorCode::InternalError(
1576 "failed to parse host from meta store endpoint".to_owned(),
1577 )
1578 })?
1579 .to_owned();
1580 let meta_store_port = meta_store_endpoint.port().ok_or_else(|| {
1581 ErrorCode::InternalError(
1582 "failed to parse port from meta store endpoint".to_owned(),
1583 )
1584 })?;
1585 let meta_store_database = meta_store_endpoint
1586 .path()
1587 .trim_start_matches('/')
1588 .to_owned();
1589
1590 let Ok(meta_backend) = MetaBackend::from_str(&meta_store_backend, true) else {
1591 bail!("failed to parse meta backend: {}", meta_store_backend);
1592 };
1593
1594 let catalog_uri = match meta_backend {
1595 MetaBackend::Postgres => {
1596 format!(
1597 "jdbc:postgresql://{}:{}/{}",
1598 meta_store_host, meta_store_port, meta_store_database
1599 )
1600 }
1601 MetaBackend::Mysql => {
1602 format!(
1603 "jdbc:mysql://{}:{}/{}",
1604 meta_store_host, meta_store_port, meta_store_database
1605 )
1606 }
1607 MetaBackend::Sqlite | MetaBackend::Sql | MetaBackend::Mem => {
1608 bail!(
1609 "Unsupported meta backend for iceberg engine table: {}",
1610 meta_store_backend
1611 );
1612 }
1613 };
1614
1615 with_common.insert("catalog.type".to_owned(), "jdbc".to_owned());
1616 with_common.insert("catalog.uri".to_owned(), catalog_uri);
1617 with_common.insert("catalog.jdbc.user".to_owned(), meta_store_user);
1618 with_common.insert("catalog.jdbc.password".to_owned(), meta_store_password);
1619 with_common.insert("catalog.name".to_owned(), iceberg_catalog_name);
1620 }
1621
1622 with_common
1623 } else {
1624 return Err(RwError::from(ErrorCode::InvalidParameterValue(
1625 "Only iceberg connection could be used in iceberg engine".to_owned(),
1626 )));
1627 }
1628 } else {
1629 return Err(RwError::from(ErrorCode::InvalidParameterValue(
1630 "Private Link Service has been deprecated. Please create a new connection instead."
1631 .to_owned(),
1632 )));
1633 }
1634 };
1635
1636 let mut pks = table
1639 .pk_column_names()
1640 .iter()
1641 .map(|c| c.to_string())
1642 .collect::<Vec<String>>();
1643
1644 if pks.len() == 1 && pks[0].eq(ROW_ID_COLUMN_NAME) {
1646 pks = vec![RISINGWAVE_ICEBERG_ROW_ID.to_owned()];
1647 }
1648
1649 let sink_from = CreateSink::From(table_name.clone());
1650
1651 let mut sink_name = table_name.clone();
1652 *sink_name.0.last_mut().unwrap() = Ident::from(
1653 (ICEBERG_SINK_PREFIX.to_owned() + &sink_name.0.last().unwrap().real_value()).as_str(),
1654 );
1655 let create_sink_stmt = CreateSinkStatement {
1656 or_replace: false,
1657 if_not_exists: false,
1658 sink_name,
1659 with_properties: WithProperties(vec![]),
1660 sink_from,
1661 columns: vec![],
1662 emit_mode: None,
1663 sink_schema: None,
1664 into_table_name: None,
1665 };
1666
1667 let mut sink_handler_args = handler_args.clone();
1668
1669 let sink_with = build_iceberg_engine_sink_options(
1670 with_common.clone(),
1671 &handler_args.with_options,
1672 &table,
1673 &pks,
1674 )?;
1675
1676 if let Some(source) = source.as_mut() {
1677 source
1678 .with_properties
1679 .retain(|key, _| !is_iceberg_engine_option(key));
1680 }
1681
1682 sink_handler_args.with_options =
1702 WithOptions::new(sink_with, Default::default(), connection_ref.clone());
1703 let SinkPlanContext {
1704 sink_plan,
1705 sink_catalog,
1706 ..
1707 } = gen_sink_plan(sink_handler_args, create_sink_stmt, None, true).await?;
1708 let sink_graph = build_graph(sink_plan, Some(GraphJobType::Sink))?;
1709
1710 let mut source_name = table_name.clone();
1711 *source_name.0.last_mut().unwrap() = Ident::from(
1712 (ICEBERG_SOURCE_PREFIX.to_owned() + &source_name.0.last().unwrap().real_value()).as_str(),
1713 );
1714 let create_source_stmt = CreateSourceStatement {
1715 temporary: false,
1716 if_not_exists: false,
1717 columns: vec![],
1718 source_name,
1719 wildcard_idx: Some(0),
1720 constraints: vec![],
1721 with_properties: WithProperties(vec![]),
1722 format_encode: CompatibleFormatEncode::V2(FormatEncodeOptions::none()),
1723 source_watermarks: vec![],
1724 include_column_options: vec![],
1725 cdc_table_info: None,
1726 };
1727
1728 let mut source_handler_args = handler_args.clone();
1729 let source_with = with_common;
1730 source_handler_args.with_options =
1731 WithOptions::new(source_with, Default::default(), connection_ref);
1732
1733 let overwrite_options = OverwriteOptions::new(&mut source_handler_args)?;
1734 let format_encode = create_source_stmt.format_encode.into_v2_with_warning();
1735 let (with_properties, ingest_mode) =
1736 bind_connector_props(&source_handler_args, &format_encode, true)?;
1737
1738 let (iceberg_catalog, table_identifier, table_created) = {
1741 let sink_param = SinkParam::try_from_sink_catalog(sink_catalog.clone())?;
1742 let iceberg_sink = IcebergSink::try_from(sink_param)?;
1743 let table_created = iceberg_sink.create_table_if_not_exists().await?;
1744
1745 let iceberg_catalog = iceberg_sink.config.create_catalog().await?;
1746 let table_identifier = iceberg_sink.config.full_table_name()?;
1747 (iceberg_catalog, table_identifier, table_created)
1748 };
1749
1750 let create_source_type = CreateSourceType::for_newly_created(&session, &*with_properties);
1751 let (columns_from_resolve_source, source_info) = bind_columns_from_source(
1752 &session,
1753 &format_encode,
1754 Either::Left(&with_properties),
1755 create_source_type,
1756 )
1757 .await?;
1758 let mut col_id_gen = ColumnIdGenerator::new_initial();
1759
1760 let iceberg_source_catalog = bind_create_source_or_table_with_connector(
1761 source_handler_args,
1762 create_source_stmt.source_name,
1763 format_encode,
1764 with_properties,
1765 &create_source_stmt.columns,
1766 create_source_stmt.constraints,
1767 create_source_stmt.wildcard_idx,
1768 create_source_stmt.source_watermarks,
1769 columns_from_resolve_source,
1770 source_info,
1771 create_source_stmt.include_column_options,
1772 &mut col_id_gen,
1773 create_source_type,
1774 overwrite_options.source_rate_limit,
1775 SqlColumnStrategy::FollowChecked,
1776 ingest_mode,
1777 )
1778 .await?;
1779
1780 let _ = Jvm::get_or_init()?;
1783
1784 let catalog_writer = session.catalog_writer()?;
1785 let action = match job_type {
1786 TableJobType::SharedCdcSource => LongRunningNotificationAction::MonitorBackfillJob,
1787 _ => LongRunningNotificationAction::DiagnoseBarrierLatency,
1788 };
1789 let res = execute_with_long_running_notification(
1790 catalog_writer.create_iceberg_table(
1791 PbTableJobInfo {
1792 source,
1793 table: Some(table.to_prost()),
1794 fragment_graph: Some(graph),
1795 job_type: job_type as _,
1796 },
1797 PbSinkJobInfo {
1798 sink: Some(sink_catalog.to_proto()),
1799 fragment_graph: Some(sink_graph),
1800 },
1801 iceberg_source_catalog.to_prost(),
1802 if_not_exists,
1803 ),
1804 &session,
1805 "CREATE TABLE",
1806 action,
1807 )
1808 .await;
1809
1810 if res.is_err() {
1811 if table_created {
1814 let _ = iceberg_catalog
1815 .drop_table(&table_identifier)
1816 .await
1817 .inspect_err(|err| {
1818 tracing::error!(
1819 "failed to drop iceberg table {} after create iceberg engine table failed: {}",
1820 table_identifier,
1821 err.as_report()
1822 );
1823 });
1824 }
1825 res?
1826 }
1827
1828 Ok(())
1829}
1830
1831pub fn check_create_table_with_source(
1832 with_options: &WithOptions,
1833 format_encode: Option<FormatEncodeOptions>,
1834 include_column_options: &IncludeOption,
1835 cdc_table_info: &Option<CdcTableInfo>,
1836) -> Result<Option<FormatEncodeOptions>> {
1837 if cdc_table_info.is_some() {
1839 return Ok(format_encode);
1840 }
1841 let defined_source = with_options.is_source_connector();
1842
1843 if !include_column_options.is_empty() && !defined_source {
1844 return Err(ErrorCode::InvalidInputSyntax(
1845 "INCLUDE should be used with a connector".to_owned(),
1846 )
1847 .into());
1848 }
1849 if defined_source {
1850 format_encode.as_ref().ok_or_else(|| {
1851 ErrorCode::InvalidInputSyntax("Please specify a source schema using FORMAT".to_owned())
1852 })?;
1853 }
1854 Ok(format_encode)
1855}
1856
1857fn ensure_partition_columns_are_prefix_of_primary_key(
1858 partition_columns: &[String],
1859 primary_key_columns: &[String],
1860) -> std::result::Result<(), String> {
1861 if partition_columns.len() > primary_key_columns.len() {
1862 return Err("Partition columns cannot be longer than primary key columns.".to_owned());
1863 }
1864
1865 for (i, partition_col) in partition_columns.iter().enumerate() {
1866 if primary_key_columns.get(i) != Some(partition_col) {
1867 return Err(format!(
1868 "Partition column '{}' is not a prefix of the primary key.",
1869 partition_col
1870 ));
1871 }
1872 }
1873
1874 Ok(())
1875}
1876
1877#[allow(clippy::too_many_arguments)]
1878pub async fn generate_stream_graph_for_replace_table(
1879 _session: &Arc<SessionImpl>,
1880 table_name: ObjectName,
1881 original_catalog: &Arc<TableCatalog>,
1882 handler_args: HandlerArgs,
1883 statement: Statement,
1884 col_id_gen: ColumnIdGenerator,
1885 sql_column_strategy: SqlColumnStrategy,
1886) -> Result<(
1887 StreamFragmentGraph,
1888 TableCatalog,
1889 Option<SourceCatalog>,
1890 TableJobType,
1891)> {
1892 let Statement::CreateTable {
1893 columns,
1894 constraints,
1895 source_watermarks,
1896 append_only,
1897 on_conflict,
1898 with_version_columns,
1899 wildcard_idx,
1900 cdc_table_info,
1901 format_encode,
1902 include_column_options,
1903 engine,
1904 with_options,
1905 ..
1906 } = statement
1907 else {
1908 panic!("unexpected statement type: {:?}", statement);
1909 };
1910
1911 let format_encode = format_encode
1912 .clone()
1913 .map(|format_encode| format_encode.into_v2_with_warning());
1914
1915 let engine = match engine {
1916 risingwave_sqlparser::ast::Engine::Hummock => Engine::Hummock,
1917 risingwave_sqlparser::ast::Engine::Iceberg => Engine::Iceberg,
1918 };
1919
1920 let is_drop_connector =
1921 original_catalog.associated_source_id().is_some() && format_encode.is_none();
1922 if is_drop_connector {
1923 debug_assert!(
1924 include_column_options.is_empty()
1925 && with_options
1926 .iter()
1927 .all(|opt| opt.name.real_value().to_lowercase() != "connector")
1928 );
1929 }
1930
1931 let props = CreateTableProps {
1932 definition: handler_args.normalized_sql.clone(),
1933 append_only,
1934 on_conflict: on_conflict.into(),
1935 with_version_columns: with_version_columns
1936 .iter()
1937 .map(|col| col.real_value())
1938 .collect(),
1939 webhook_info: original_catalog.webhook_info.clone(),
1940 engine,
1941 };
1942
1943 let ((plan, mut source, mut table), job_type) = match (format_encode, cdc_table_info.as_ref()) {
1944 (Some(format_encode), None) => (
1945 gen_create_table_plan_with_source(
1946 handler_args,
1947 ExplainOptions::default(),
1948 table_name,
1949 columns,
1950 wildcard_idx,
1951 constraints,
1952 format_encode,
1953 source_watermarks,
1954 col_id_gen,
1955 include_column_options,
1956 props,
1957 sql_column_strategy,
1958 )
1959 .await?,
1960 TableJobType::General,
1961 ),
1962 (None, None) => {
1963 let context = OptimizerContext::from_handler_args(handler_args)?;
1964 let (plan, table) = gen_create_table_plan(
1965 context,
1966 table_name,
1967 columns,
1968 constraints,
1969 col_id_gen,
1970 source_watermarks,
1971 props,
1972 true,
1973 )?;
1974 ((plan, None, table), TableJobType::General)
1975 }
1976 (None, Some(cdc_table)) => {
1977 sanity_check_for_table_on_cdc_source(
1978 append_only,
1979 &columns,
1980 &wildcard_idx,
1981 &constraints,
1982 &source_watermarks,
1983 )?;
1984
1985 let session = &handler_args.session;
1986 let (source, resolved_table_name) =
1987 get_source_and_resolved_table_name(session, cdc_table.clone(), table_name.clone())?;
1988
1989 let (cdc_with_options, external_table_name) = derive_connect_properties_for_cdc_table(
1990 &source.with_properties,
1991 cdc_table.external_table_name.clone(),
1992 )?;
1993
1994 let (column_catalogs, pk_names) = bind_cdc_table_schema(&columns, &constraints, true)?;
1995 let pk_comparisons = Box::pin(bind_cdc_pk_comparisons_externally(
1996 cdc_with_options.clone(),
1997 &pk_names,
1998 ))
1999 .await?;
2000
2001 reject_pk_filtered_by_debezium_column_filter(&pk_names, &cdc_with_options)?;
2004
2005 let context: OptimizerContextRef =
2006 OptimizerContext::new(handler_args, ExplainOptions::default())?.into();
2007 let (plan, table) = gen_create_table_plan_for_cdc_table(
2008 context,
2009 source,
2010 external_table_name,
2011 columns,
2012 source_watermarks,
2013 column_catalogs,
2014 pk_names,
2015 pk_comparisons,
2016 cdc_with_options,
2017 col_id_gen,
2018 on_conflict,
2019 with_version_columns
2020 .iter()
2021 .map(|col| col.real_value())
2022 .collect(),
2023 include_column_options,
2024 table_name,
2025 resolved_table_name,
2026 original_catalog.database_id,
2027 original_catalog.schema_id,
2028 original_catalog.id(),
2029 engine,
2030 )?;
2031
2032 ((plan, None, table), TableJobType::SharedCdcSource)
2033 }
2034 (Some(_), Some(_)) => {
2035 return Err(ErrorCode::NotSupported(
2036 "Data format and encoding format doesn't apply to table created from a CDC source"
2037 .into(),
2038 "Remove the FORMAT and ENCODE specification".into(),
2039 )
2040 .into());
2041 }
2042 };
2043
2044 if table.pk_column_ids() != original_catalog.pk_column_ids() {
2045 Err(ErrorCode::InvalidInputSyntax(
2046 "alter primary key of table is not supported".to_owned(),
2047 ))?
2048 }
2049
2050 let graph = build_graph(plan, Some(GraphJobType::Table))?;
2051
2052 table.id = original_catalog.id();
2054 if !is_drop_connector && let Some(source_id) = original_catalog.associated_source_id() {
2055 table.associated_source_id = Some(source_id);
2056
2057 let source = source.as_mut().unwrap();
2058 source.id = source_id;
2059 source.associated_table_id = Some(table.id());
2060 }
2061
2062 Ok((graph, table, source, job_type))
2063}
2064
2065fn get_source_and_resolved_table_name(
2066 session: &Arc<SessionImpl>,
2067 cdc_table: CdcTableInfo,
2068 table_name: ObjectName,
2069) -> Result<(Arc<SourceCatalog>, String)> {
2070 let db_name = &session.database();
2071 let (_, resolved_table_name) = Binder::resolve_schema_qualified_name(db_name, &table_name)?;
2072
2073 let (source_schema, source_name) =
2074 Binder::resolve_schema_qualified_name(db_name, &cdc_table.source_name)?;
2075
2076 let source = {
2077 let catalog_reader = session.env().catalog_reader().read_guard();
2078 let schema_name = source_schema.unwrap_or(DEFAULT_SCHEMA_NAME.to_owned());
2079 let (source, _) = catalog_reader.get_source_by_name(
2080 db_name,
2081 SchemaPath::Name(schema_name.as_str()),
2082 source_name.as_str(),
2083 )?;
2084 source.clone()
2085 };
2086 check_cdc_source_select_privilege(session, &source)?;
2087
2088 Ok((source, resolved_table_name))
2089}
2090
2091pub(crate) fn check_cdc_source_select_privilege(
2092 session: &SessionImpl,
2093 source: &SourceCatalog,
2094) -> Result<()> {
2095 session.check_privileges(&[ObjectCheckItem::new(
2096 source.owner,
2097 AclMode::Select,
2098 source.name.clone(),
2099 source.id,
2100 )])
2101}
2102
2103fn bind_webhook_info(
2105 session: &Arc<SessionImpl>,
2106 column_defs: &[ColumnDef],
2107 webhook_info: WebhookSourceInfo,
2108) -> Result<PbWebhookSourceInfo> {
2109 let WebhookSourceInfo {
2110 secret_ref,
2111 signature_expr,
2112 wait_for_persistence,
2113 is_batched,
2114 } = webhook_info;
2115
2116 for column in column_defs {
2117 for option_def in &column.options {
2118 match option_def.option {
2119 ColumnOption::Null => {}
2120 ColumnOption::GeneratedColumns(_) => {
2121 return Err(ErrorCode::InvalidInputSyntax(
2122 "generated columns are not supported for webhook tables".to_owned(),
2123 )
2124 .into());
2125 }
2126 ColumnOption::DefaultValue(_) | ColumnOption::DefaultValueInternal { .. } => {
2127 return Err(ErrorCode::InvalidInputSyntax(
2128 "default values are not supported for webhook tables".to_owned(),
2129 )
2130 .into());
2131 }
2132 ColumnOption::NotNull
2133 | ColumnOption::Unique { .. }
2134 | ColumnOption::ForeignKey { .. }
2135 | ColumnOption::Check(_)
2136 | ColumnOption::DialectSpecific(_) => {
2137 return Err(ErrorCode::InvalidInputSyntax(
2138 "only NULL column option is supported for webhook tables".to_owned(),
2139 )
2140 .into());
2141 }
2142 }
2143 }
2144 }
2145
2146 let (pb_secret_ref, secret_name) = if let Some(secret_ref) = secret_ref {
2148 let db_name = &session.database();
2149 let (schema_name, secret_name) =
2150 Binder::resolve_schema_qualified_name(db_name, &secret_ref.secret_name)?;
2151 let secret_catalog = session.get_secret_by_name(schema_name, &secret_name)?;
2152 (
2153 Some(PbSecretRef {
2154 secret_id: secret_catalog.id,
2155 ref_as: match secret_ref.ref_as {
2156 SecretRefAsType::Text => PbRefAsType::Text,
2157 SecretRefAsType::File => PbRefAsType::File,
2158 }
2159 .into(),
2160 }),
2161 Some(secret_name),
2162 )
2163 } else {
2164 (None, None)
2165 };
2166
2167 let signature_expr = if let Some(signature_expr) = signature_expr {
2168 let payload_name = if column_defs.len() == 1
2169 && column_defs[0].data_type.as_ref() == Some(&AstDataType::Jsonb)
2170 {
2171 column_defs[0].name.real_value()
2172 } else {
2173 WEBHOOK_PAYLOAD_FIELD_NAME.to_owned()
2174 };
2175 let secure_compare_context = SecureCompareContext {
2176 payload_name,
2177 secret_name,
2178 };
2179 let mut binder = Binder::new_for_ddl(session).with_secure_compare(secure_compare_context);
2180 let expr = binder.bind_expr(&signature_expr)?;
2181
2182 if expr.as_function_call().is_none()
2184 || expr.as_function_call().unwrap().func_type()
2185 != crate::optimizer::plan_node::generic::ExprType::SecureCompare
2186 {
2187 return Err(ErrorCode::InvalidInputSyntax(
2188 "The signature verification function must be SECURE_COMPARE()".to_owned(),
2189 )
2190 .into());
2191 }
2192
2193 Some(expr.to_expr_proto())
2194 } else {
2195 session.notice_to_user(
2196 "VALIDATE clause is strongly recommended for safety or production usages",
2197 );
2198 None
2199 };
2200
2201 let pb_webhook_info = PbWebhookSourceInfo {
2202 secret_ref: pb_secret_ref,
2203 signature_expr,
2204 wait_for_persistence,
2205 is_batched,
2206 };
2207
2208 Ok(pb_webhook_info)
2209}
2210
2211#[cfg(test)]
2212mod tests {
2213 use risingwave_common::catalog::{
2214 DEFAULT_DATABASE_NAME, ROW_ID_COLUMN_NAME, RW_TIMESTAMP_COLUMN_NAME,
2215 };
2216 use risingwave_common::types::StructType;
2217
2218 use super::*;
2219 use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
2220
2221 #[tokio::test]
2222 async fn test_iceberg_engine_primary_key_types() {
2223 let frontend = LocalFrontend::new(Default::default()).await;
2224 for (primary_key, append_only, pk_index, supported) in [
2225 ("id, period", false, false, false),
2226 ("id, items", false, false, false),
2227 ("id, mapping", false, false, false),
2228 ("id, f", false, false, false),
2229 ("id, d", false, false, false),
2230 ("id", false, false, true),
2231 ("id, ts", false, false, true),
2232 ("", false, false, true),
2233 ("id, period", true, false, true),
2234 ("id, period", false, true, true),
2235 ] {
2236 let pk_clause = if primary_key.is_empty() {
2237 String::new()
2238 } else {
2239 format!(", PRIMARY KEY ({primary_key})")
2240 };
2241 frontend
2242 .run_sql(&format!(
2243 "CREATE TABLE t (id INT, period STRUCT<start_ts TIMESTAMPTZ, end_ts TIMESTAMPTZ>, \
2244 items INT[], mapping MAP(INT, INT), f REAL, d DOUBLE PRECISION, ts TIMESTAMPTZ \
2245 {pk_clause}) {}",
2246 if append_only { "APPEND ONLY" } else { "" }
2247 ))
2248 .await
2249 .unwrap();
2250 let session = frontend.session_ref();
2251 let table = session
2252 .env()
2253 .catalog_reader()
2254 .read_guard()
2255 .get_created_table_by_name(
2256 DEFAULT_DATABASE_NAME,
2257 SchemaPath::Name(DEFAULT_SCHEMA_NAME),
2258 "t",
2259 )
2260 .unwrap()
2261 .0
2262 .clone();
2263 let pks = table
2264 .pk_column_names()
2265 .into_iter()
2266 .map(str::to_owned)
2267 .collect_vec();
2268 for write_mode in ["merge-on-read", "copy-on-write"] {
2269 if write_mode == "copy-on-write" && (append_only || pk_index) {
2270 continue;
2271 }
2272 let options = BTreeMap::from([
2273 ("connector".to_owned(), "iceberg".to_owned()),
2274 ("catalog.type".to_owned(), "storage".to_owned()),
2275 (
2276 "warehouse.path".to_owned(),
2277 "s3://test/warehouse".to_owned(),
2278 ),
2279 ("database.name".to_owned(), "public".to_owned()),
2280 ("table.name".to_owned(), "t".to_owned()),
2281 ("write_mode".to_owned(), write_mode.to_owned()),
2282 ("enable_pk_index".to_owned(), pk_index.to_string()),
2283 ]);
2284 let result = build_iceberg_engine_sink_options(
2285 options,
2286 &WithOptions::default(),
2287 &table,
2288 &pks,
2289 );
2290 if supported {
2291 result.unwrap();
2292 } else {
2293 let err = result.unwrap_err().to_report_string();
2294 let column = primary_key.split(", ").last().unwrap();
2295 assert!(
2296 err.contains(&format!(
2297 "Iceberg engine table primary key column \"{column}\" has unsupported type"
2298 )),
2299 "{write_mode}: {err}"
2300 );
2301 }
2302 }
2303 frontend.run_sql("DROP TABLE t").await.unwrap();
2304 }
2305 }
2306
2307 #[tokio::test]
2308 async fn test_cdc_relations_require_select_privilege_on_source() {
2309 let frontend = LocalFrontend::new(Default::default()).await;
2310 frontend
2313 .run_sql(
2314 r#"
2315 CREATE SOURCE cdc_source WITH (
2316 connector = 'postgres-cdc',
2317 hostname = 'localhost',
2318 port = '5432',
2319 username = 'postgres',
2320 password = '',
2321 database.name = 'db'
2322 ) FORMAT PLAIN ENCODE JSON
2323 "#,
2324 )
2325 .await
2326 .unwrap();
2327 frontend.run_sql("CREATE USER cdc_user").await.unwrap();
2328 frontend
2329 .run_sql("GRANT CREATE ON SCHEMA public TO cdc_user")
2330 .await
2331 .unwrap();
2332
2333 let user_id = frontend
2334 .session_ref()
2335 .env()
2336 .user_info_reader()
2337 .read_guard()
2338 .get_user_by_name("cdc_user")
2339 .unwrap()
2340 .id;
2341 let user_session = frontend.session_user_ref(
2342 DEFAULT_DATABASE_NAME.to_owned(),
2343 "cdc_user".to_owned(),
2344 user_id,
2345 );
2346 let create_table =
2347 "CREATE TABLE cdc_table (id INT PRIMARY KEY) FROM cdc_source TABLE 'public.t'";
2348 let create_table_source = "CREATE SOURCE cdc_table_source (id INT PRIMARY KEY) \
2349 WITH (snapshot = 'false') FROM cdc_source TABLE 'public.t'";
2350
2351 for sql in [create_table, create_table_source] {
2352 let err = frontend
2353 .run_sql_with_session(user_session.clone(), sql)
2354 .await
2355 .unwrap_err();
2356 assert!(
2357 err.to_string()
2358 .contains("permission denied for source \"cdc_source\": \"SELECT\""),
2359 "{err:?}"
2360 );
2361 }
2362
2363 frontend
2364 .run_sql("GRANT SELECT ON SOURCE cdc_source TO cdc_user")
2365 .await
2366 .unwrap();
2367 frontend
2368 .run_sql_with_session(user_session.clone(), create_table)
2369 .await
2370 .unwrap();
2371 frontend
2372 .run_sql_with_session(user_session.clone(), create_table_source)
2373 .await
2374 .unwrap();
2375
2376 let table_source = {
2377 let catalog_reader = frontend.session_ref().env().catalog_reader().read_guard();
2378 catalog_reader
2379 .get_source_by_name(
2380 DEFAULT_DATABASE_NAME,
2381 SchemaPath::Name("public"),
2382 "cdc_table_source",
2383 )
2384 .unwrap()
2385 .0
2386 .clone()
2387 };
2388 assert_eq!(table_source.with_properties.len(), 2);
2389 assert_eq!(
2390 table_source.with_properties.get_connector().as_deref(),
2391 Some("postgres-cdc")
2392 );
2393 assert_eq!(
2394 table_source
2395 .with_properties
2396 .get("snapshot")
2397 .map(String::as_str),
2398 Some("false")
2399 );
2400 assert!(table_source.with_properties.as_secret().is_empty());
2401 let external_table = table_source.info.external_table.as_ref().unwrap();
2405 assert!(!external_table.connect_properties.is_empty());
2406
2407 let err = frontend
2408 .run_sql_with_session(
2409 user_session.clone(),
2410 "SELECT * FROM postgres_query('cdc_table_source', 'SELECT 1')",
2411 )
2412 .await
2413 .unwrap_err();
2414 assert!(
2415 err.to_report_string()
2416 .contains("only accept shared CDC sources, not CDC table sources"),
2417 "{err:?}"
2418 );
2419
2420 frontend
2421 .run_sql("REVOKE SELECT ON SOURCE cdc_source FROM cdc_user")
2422 .await
2423 .unwrap();
2424 let err = frontend
2425 .run_sql_with_session(user_session, "ALTER TABLE cdc_table ADD COLUMN value INT")
2426 .await
2427 .unwrap_err();
2428 assert!(
2429 err.to_string()
2430 .contains("permission denied for source \"cdc_source\": \"SELECT\""),
2431 "{err:?}"
2432 );
2433 }
2434
2435 #[tokio::test]
2436 async fn test_create_table_handler() {
2437 let sql =
2438 "create table t (v1 smallint, v2 struct<v3 bigint, v4 float, v5 double>) append only;";
2439 let frontend = LocalFrontend::new(Default::default()).await;
2440 frontend.run_sql(sql).await.unwrap();
2441
2442 let session = frontend.session_ref();
2443 let catalog_reader = session.env().catalog_reader().read_guard();
2444 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
2445
2446 let (table, _) = catalog_reader
2448 .get_created_table_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
2449 .unwrap();
2450 assert_eq!(table.name(), "t");
2451
2452 let columns = table
2453 .columns
2454 .iter()
2455 .map(|col| (col.name(), col.data_type().clone()))
2456 .collect::<HashMap<&str, DataType>>();
2457
2458 let expected_columns = maplit::hashmap! {
2459 ROW_ID_COLUMN_NAME => DataType::Serial,
2460 "v1" => DataType::Int16,
2461 "v2" => StructType::new(
2462 vec![("v3", DataType::Int64),("v4", DataType::Float64),("v5", DataType::Float64)],
2463 )
2464 .with_ids([3, 4, 5].map(ColumnId::new))
2465 .into(),
2466 RW_TIMESTAMP_COLUMN_NAME => DataType::Timestamptz,
2467 };
2468
2469 assert_eq!(columns, expected_columns, "{columns:#?}");
2470 }
2471
2472 #[tokio::test]
2473 async fn test_create_webhook_table_with_arbitrary_columns() {
2474 let frontend = LocalFrontend::new(Default::default()).await;
2475 frontend
2476 .run_sql("create schema ingest_schema;")
2477 .await
2478 .unwrap();
2479 frontend
2480 .run_sql(
2481 r#"
2482 create table ingest_schema.orders (
2483 id int,
2484 customer_name varchar,
2485 amount double precision,
2486 primary key (id)
2487 ) with (
2488 connector = 'webhook'
2489 ) validate as secure_compare(
2490 headers->>'x-rw-signature',
2491 'sha256=' || encode(hmac('webhook-secret', payload, 'sha256'), 'hex')
2492 );
2493 "#,
2494 )
2495 .await
2496 .unwrap();
2497
2498 let session = frontend.session_ref();
2499 let catalog_reader = session.env().catalog_reader().read_guard();
2500 let (table, _) = catalog_reader
2501 .get_created_table_by_name(
2502 DEFAULT_DATABASE_NAME,
2503 SchemaPath::Name("ingest_schema"),
2504 "orders",
2505 )
2506 .unwrap();
2507
2508 assert!(table.webhook_info.is_some());
2509 assert_eq!(
2510 table
2511 .columns
2512 .iter()
2513 .filter(|column| column.can_dml())
2514 .count(),
2515 3
2516 );
2517 }
2518
2519 #[tokio::test]
2520 async fn test_create_webhook_table_uses_single_jsonb_column_name_in_validate() {
2521 let frontend = LocalFrontend::new(Default::default()).await;
2522 frontend
2523 .run_sql(
2524 r#"
2525 create table webhook_single_column (
2526 body jsonb
2527 ) with (
2528 connector = 'webhook'
2529 ) validate as secure_compare(
2530 headers->>'x-rw-signature',
2531 'sha256=' || encode(hmac('webhook-secret', body, 'sha256'), 'hex')
2532 );
2533 "#,
2534 )
2535 .await
2536 .unwrap();
2537 }
2538
2539 #[tokio::test]
2540 async fn test_create_webhook_table_with_generated_columns() {
2541 let frontend = LocalFrontend::new(Default::default()).await;
2542 let err = frontend
2543 .run_sql(
2544 r#"
2545 create table webhook_generated_columns (
2546 id int,
2547 amount double precision,
2548 amount_with_fee double precision as amount + 1.0
2549 ) with (
2550 connector = 'webhook'
2551 );
2552 "#,
2553 )
2554 .await
2555 .unwrap_err();
2556
2557 assert!(
2558 err.to_string()
2559 .contains("generated columns are not supported for webhook tables"),
2560 "{err:?}"
2561 );
2562 }
2563
2564 #[tokio::test]
2565 async fn test_create_webhook_table_with_default_value() {
2566 let frontend = LocalFrontend::new(Default::default()).await;
2567 let err = frontend
2568 .run_sql(
2569 r#"
2570 create table webhook_default_value (
2571 id int default 42,
2572 amount double precision
2573 ) with (
2574 connector = 'webhook'
2575 );
2576 "#,
2577 )
2578 .await
2579 .unwrap_err();
2580
2581 assert!(
2582 err.to_string()
2583 .contains("default values are not supported for webhook tables"),
2584 "{err:?}"
2585 );
2586 }
2587
2588 #[tokio::test]
2589 async fn test_create_webhook_table_with_not_null_option() {
2590 let frontend = LocalFrontend::new(Default::default()).await;
2591 let err = frontend
2592 .run_sql(
2593 r#"
2594 create table webhook_not_null (
2595 id int not null,
2596 amount double precision
2597 ) with (
2598 connector = 'webhook'
2599 );
2600 "#,
2601 )
2602 .await
2603 .unwrap_err();
2604
2605 assert!(
2606 err.to_string()
2607 .contains("only NULL column option is supported for webhook tables"),
2608 "{err:?}"
2609 );
2610 }
2611
2612 #[test]
2613 fn test_bind_primary_key() {
2614 for (sql, expected) in [
2617 ("create table t (v1 int, v2 int)", Ok(&[0] as &[_])),
2618 ("create table t (v1 int primary key, v2 int)", Ok(&[1])),
2619 ("create table t (v1 int, v2 int primary key)", Ok(&[2])),
2620 (
2621 "create table t (v1 int primary key, v2 int primary key)",
2622 Err("multiple primary keys are not allowed"),
2623 ),
2624 (
2625 "create table t (v1 int primary key primary key, v2 int)",
2626 Err("multiple primary keys are not allowed"),
2627 ),
2628 (
2629 "create table t (v1 int, v2 int, primary key (v1))",
2630 Ok(&[1]),
2631 ),
2632 (
2633 "create table t (v1 int, primary key (v2), v2 int)",
2634 Ok(&[2]),
2635 ),
2636 (
2637 "create table t (primary key (v2, v1), v1 int, v2 int)",
2638 Ok(&[2, 1]),
2639 ),
2640 (
2641 "create table t (v1 int, primary key (v1), v2 int, primary key (v1))",
2642 Err("multiple primary keys are not allowed"),
2643 ),
2644 (
2645 "create table t (v1 int primary key, primary key (v1), v2 int)",
2646 Err("multiple primary keys are not allowed"),
2647 ),
2648 (
2649 "create table t (v1 int, primary key (V3), v2 int)",
2650 Err("column \"v3\" named in key does not exist"),
2651 ),
2652 ] {
2653 let mut ast = risingwave_sqlparser::parser::Parser::parse_sql(sql).unwrap();
2654 let risingwave_sqlparser::ast::Statement::CreateTable {
2655 columns: column_defs,
2656 constraints,
2657 ..
2658 } = ast.remove(0)
2659 else {
2660 panic!("test case should be create table")
2661 };
2662 let actual: Result<_> = (|| {
2663 let mut columns = bind_sql_columns(&column_defs, false)?;
2664 let mut col_id_gen = ColumnIdGenerator::new_initial();
2665 for c in &mut columns {
2666 col_id_gen.generate(c)?;
2667 }
2668
2669 let pk_names =
2670 bind_sql_pk_names(&column_defs, bind_table_constraints(&constraints)?)?;
2671 let (_, pk_column_ids, _) =
2672 bind_pk_and_row_id_on_relation(columns, pk_names, true)?;
2673 Ok(pk_column_ids)
2674 })();
2675 match (expected, actual) {
2676 (Ok(expected), Ok(actual)) => assert_eq!(
2677 expected.iter().copied().map(ColumnId::new).collect_vec(),
2678 actual,
2679 "sql: {sql}"
2680 ),
2681 (Ok(_), Err(actual)) => panic!("sql: {sql}\nunexpected error: {actual:?}"),
2682 (Err(_), Ok(actual)) => panic!("sql: {sql}\nexpects error but got: {actual:?}"),
2683 (Err(expected), Err(actual)) => assert!(
2684 actual.to_string().contains(expected),
2685 "sql: {sql}\nexpected: {expected:?}\nactual: {actual:?}"
2686 ),
2687 }
2688 }
2689 }
2690
2691 #[tokio::test]
2692 async fn test_duplicate_props_options() {
2693 let proto_file = create_proto_file(PROTO_FILE_DATA);
2694 let sql = format!(
2695 r#"CREATE TABLE t
2696 WITH (
2697 connector = 'kinesis',
2698 aws.region='user_test_topic',
2699 endpoint='172.10.1.1:9090,172.10.1.2:9090',
2700 aws.credentials.access_key_id = 'your_access_key_1',
2701 aws.credentials.secret_access_key = 'your_secret_key_1'
2702 )
2703 FORMAT PLAIN ENCODE PROTOBUF (
2704 message = '.test.TestRecord',
2705 aws.credentials.access_key_id = 'your_access_key_2',
2706 aws.credentials.secret_access_key = 'your_secret_key_2',
2707 schema.location = 'file://{}',
2708 )"#,
2709 proto_file.path().to_str().unwrap()
2710 );
2711 let frontend = LocalFrontend::new(Default::default()).await;
2712 frontend.run_sql(sql).await.unwrap();
2713
2714 let session = frontend.session_ref();
2715 let catalog_reader = session.env().catalog_reader().read_guard();
2716 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
2717
2718 let (source, _) = catalog_reader
2720 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "t")
2721 .unwrap();
2722 assert_eq!(source.name, "t");
2723
2724 assert_eq!(
2726 source
2727 .info
2728 .format_encode_options
2729 .get("aws.credentials.access_key_id")
2730 .unwrap(),
2731 "your_access_key_2"
2732 );
2733 assert_eq!(
2734 source
2735 .info
2736 .format_encode_options
2737 .get("aws.credentials.secret_access_key")
2738 .unwrap(),
2739 "your_secret_key_2"
2740 );
2741
2742 assert_eq!(
2744 source
2745 .with_properties
2746 .get("aws.credentials.access_key_id")
2747 .unwrap(),
2748 "your_access_key_1"
2749 );
2750 assert_eq!(
2751 source
2752 .with_properties
2753 .get("aws.credentials.secret_access_key")
2754 .unwrap(),
2755 "your_secret_key_1"
2756 );
2757
2758 assert!(!source.with_properties.contains_key("schema.location"));
2760 }
2761}