1use std::sync::Arc;
16
17use either::Either;
18use itertools::Itertools;
19use pgwire::pg_response::StatementType;
20use risingwave_common::bail_not_implemented;
21use risingwave_common::catalog::{ColumnCatalog, max_column_id};
22use risingwave_connector::WithPropertiesExt;
23use risingwave_pb::catalog::StreamSourceInfo;
24use risingwave_pb::plan_common::{EncodeType, FormatType};
25use risingwave_sqlparser::ast::{
26 CompatibleFormatEncode, CreateSourceStatement, Encode, Format, FormatEncodeOptions, ObjectName,
27 SqlOption, Statement,
28};
29
30use super::create_source::{
31 generate_stream_graph_for_source, schema_has_schema_registry, validate_compatibility,
32};
33use super::util::SourceSchemaCompatExt;
34use super::{HandlerArgs, RwPgResponse};
35use crate::catalog::root_catalog::SchemaPath;
36use crate::catalog::source_catalog::SourceCatalog;
37use crate::error::{ErrorCode, Result};
38use crate::handler::create_source::{CreateSourceType, bind_columns_from_source};
39use crate::session::SessionImpl;
40use crate::utils::resolve_secret_ref_in_with_options;
41use crate::{Binder, WithOptions};
42
43fn format_type_to_format(from: FormatType) -> Option<Format> {
44 Some(match from {
45 FormatType::Unspecified => return None,
46 FormatType::Native => Format::Native,
47 FormatType::Debezium => Format::Debezium,
48 FormatType::DebeziumMongo => Format::DebeziumMongo,
49 FormatType::Maxwell => Format::Maxwell,
50 FormatType::Canal => Format::Canal,
51 FormatType::Upsert => Format::Upsert,
52 FormatType::Plain => Format::Plain,
53 FormatType::None => Format::None,
54 })
55}
56
57fn encode_type_to_encode(from: EncodeType) -> Option<Encode> {
58 Some(match from {
59 EncodeType::Unspecified => return None,
60 EncodeType::Native => Encode::Native,
61 EncodeType::Avro => Encode::Avro,
62 EncodeType::Csv => Encode::Csv,
63 EncodeType::Protobuf => Encode::Protobuf,
64 EncodeType::Json => Encode::Json,
65 EncodeType::Bytes => Encode::Bytes,
66 EncodeType::Template => Encode::Template,
67 EncodeType::Parquet => Encode::Parquet,
68 EncodeType::None => Encode::None,
69 EncodeType::Text => Encode::Text,
70 })
71}
72
73fn columns_minus(columns_a: &[ColumnCatalog], columns_b: &[ColumnCatalog]) -> Vec<ColumnCatalog> {
85 columns_a
86 .iter()
87 .filter(|col_a| {
88 !col_a.is_hidden()
89 && !col_a.is_connector_additional_column()
90 && !col_a.is_generated()
91 && !columns_b.iter().any(|col_b| {
92 col_a.name() == col_b.name() && col_a.data_type() == col_b.data_type()
93 })
94 })
95 .cloned()
96 .collect()
97}
98
99pub fn fetch_source_catalog_with_db_schema_id(
101 session: &SessionImpl,
102 name: &ObjectName,
103) -> Result<Arc<SourceCatalog>> {
104 let db_name = &session.database();
105 let (schema_name, real_source_name) = Binder::resolve_schema_qualified_name(db_name, name)?;
106 let search_path = session.config().search_path();
107 let user_name = &session.user_name();
108
109 let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
110
111 let reader = session.env().catalog_reader().read_guard();
112 let (source, schema_name) =
113 reader.get_source_by_name(db_name, schema_path, &real_source_name)?;
114
115 session.check_privilege_for_drop_alter(schema_name, &**source)?;
116
117 Ok(Arc::clone(source))
118}
119
120pub fn check_format_encode(
123 original_source: &SourceCatalog,
124 new_format_encode: &FormatEncodeOptions,
125) -> Result<()> {
126 let StreamSourceInfo {
127 format, row_encode, ..
128 } = original_source.info;
129 let (Some(old_format), Some(old_row_encode)) = (
130 format_type_to_format(FormatType::try_from(format).unwrap()),
131 encode_type_to_encode(EncodeType::try_from(row_encode).unwrap()),
132 ) else {
133 return Err(ErrorCode::NotSupported(
134 "altering a legacy source which is not created using `FORMAT .. ENCODE ..` Clause"
135 .to_owned(),
136 "try this feature by creating a fresh source".to_owned(),
137 )
138 .into());
139 };
140
141 if new_format_encode.format != old_format || new_format_encode.row_encode != old_row_encode {
142 bail_not_implemented!(
143 "the original definition is FORMAT {:?} ENCODE {:?}, and altering them is not supported yet",
144 &old_format,
145 &old_row_encode,
146 );
147 }
148
149 Ok(())
150}
151
152pub async fn refresh_sr_and_get_columns_diff(
154 original_source: &SourceCatalog,
155 format_encode: &FormatEncodeOptions,
156 session: &Arc<SessionImpl>,
157) -> Result<(StreamSourceInfo, Vec<ColumnCatalog>, Vec<ColumnCatalog>)> {
158 let mut with_properties = original_source.with_properties.clone();
159 let secret_refs = with_properties.as_secret().clone();
160 validate_compatibility(format_encode, &mut with_properties, &secret_refs)?;
161
162 if with_properties.is_cdc_connector() {
163 bail_not_implemented!("altering a cdc source is not supported");
164 }
165
166 let (Some(columns_from_resolve_source), source_info) = bind_columns_from_source(
167 session,
168 format_encode,
169 Either::Right(&with_properties),
170 CreateSourceType::for_replace(original_source),
171 )
172 .await?
173 else {
174 unreachable!("source without schema registry is rejected")
176 };
177
178 let mut added_columns = columns_minus(&columns_from_resolve_source, &original_source.columns);
179 let mut next_col_id = max_column_id(&original_source.columns).next();
181 for col in &mut added_columns {
182 col.column_desc.column_id = next_col_id;
183 next_col_id = next_col_id.next();
184 }
185 let dropped_columns = columns_minus(&original_source.columns, &columns_from_resolve_source);
186 tracing::debug!(
187 ?added_columns,
188 ?dropped_columns,
189 ?columns_from_resolve_source,
190 original_source = ?original_source.columns
191 );
192
193 Ok((source_info, added_columns, dropped_columns))
194}
195
196fn get_format_encode_from_source(source: &SourceCatalog) -> Result<FormatEncodeOptions> {
197 let stmt = source.create_sql_ast()?;
198 let Statement::CreateSource {
199 stmt: CreateSourceStatement { format_encode, .. },
200 } = stmt
201 else {
202 return Err(ErrorCode::InternalError(format!(
203 "source \"{}\" has a non-CREATE SOURCE definition",
204 source.name
205 ))
206 .into());
207 };
208 Ok(format_encode.into_v2_with_warning())
209}
210
211fn reject_associated_table(source: &SourceCatalog) -> Result<()> {
212 if source.associated_table_id.is_some() {
213 return Err(ErrorCode::NotSupported(
214 "alter table with connector using ALTER SOURCE statement".to_owned(),
215 "try to use ALTER TABLE instead".to_owned(),
216 )
217 .into());
218 }
219
220 Ok(())
221}
222
223pub async fn handler_refresh_schema(
224 handler_args: HandlerArgs,
225 name: ObjectName,
226) -> Result<RwPgResponse> {
227 let source = fetch_source_catalog_with_db_schema_id(&handler_args.session, &name)?;
228 reject_associated_table(&source)?;
229 if source.is_cdc_table_source() {
230 return Err(ErrorCode::NotSupported(
231 "refreshing the schema of a CDC table source is not supported".to_owned(),
232 "Drop and recreate the CDC table source".to_owned(),
233 )
234 .into());
235 }
236 let format_encode = get_format_encode_from_source(&source)?;
237 handle_alter_source_with_sr(handler_args, name, format_encode).await
238}
239
240pub async fn handle_alter_source_with_sr(
241 handler_args: HandlerArgs,
242 name: ObjectName,
243 format_encode: FormatEncodeOptions,
244) -> Result<RwPgResponse> {
245 let session = handler_args.session.clone();
246 let source = fetch_source_catalog_with_db_schema_id(&session, &name)?;
247 let mut source = source.as_ref().clone();
248
249 reject_associated_table(&source)?;
250 if source.is_cdc_table_source() {
251 return Err(ErrorCode::NotSupported(
252 "altering the format or schema of a CDC table source is not supported".to_owned(),
253 "Drop and recreate the CDC table source".to_owned(),
254 )
255 .into());
256 }
257
258 check_format_encode(&source, &format_encode)?;
259
260 if !schema_has_schema_registry(&format_encode) {
261 return Err(ErrorCode::NotSupported(
262 "altering a source without schema registry".to_owned(),
263 "try `ALTER SOURCE .. ADD COLUMN ...` instead".to_owned(),
264 )
265 .into());
266 }
267
268 let (source_info, added_columns, dropped_columns) =
269 refresh_sr_and_get_columns_diff(&source, &format_encode, &session).await?;
270
271 if !dropped_columns.is_empty() {
272 bail_not_implemented!(
273 "this altering statement will drop columns, which is not supported yet: {}",
274 dropped_columns
275 .iter()
276 .map(|col| format!("({}: {})", col.name(), col.data_type()))
277 .join(", ")
278 );
279 }
280
281 source.info = source_info;
282 source.columns.extend(added_columns);
283 source.definition = alter_definition_format_encode(
284 source.create_sql_ast_purified()?,
285 format_encode.row_options.clone(),
286 )?;
287
288 let (format_encode_options, format_encode_secret_ref) = resolve_secret_ref_in_with_options(
289 WithOptions::try_from(format_encode.row_options())?,
290 session.as_ref(),
291 )?
292 .into_parts();
293 source
294 .info
295 .format_encode_options
296 .extend(format_encode_options);
297
298 source
299 .info
300 .format_encode_secret_refs
301 .extend(format_encode_secret_ref);
302
303 source.version += 1;
305
306 let pb_source = source.to_prost();
307 let catalog_writer = session.catalog_writer()?;
308 if source.info.is_shared() {
309 let graph = generate_stream_graph_for_source(handler_args, source.clone())?;
310 catalog_writer.replace_source(pb_source, graph).await?
311 } else {
312 catalog_writer.alter_source(pb_source).await?;
313 }
314 Ok(RwPgResponse::empty_result(StatementType::ALTER_SOURCE))
315}
316
317pub fn alter_definition_format_encode(
319 mut stmt: Statement,
320 format_encode_options: Vec<SqlOption>,
321) -> Result<String> {
322 match &mut stmt {
323 Statement::CreateSource {
324 stmt: CreateSourceStatement { format_encode, .. },
325 }
326 | Statement::CreateTable {
327 format_encode: Some(format_encode),
328 ..
329 } => match format_encode {
330 CompatibleFormatEncode::V2(schema) => {
331 schema.row_options = format_encode_options;
332 }
333 },
334 _ => unreachable!(),
335 }
336
337 Ok(stmt.to_string())
338}
339
340#[cfg(test)]
341pub mod tests {
342 use risingwave_common::catalog::{DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME};
343 use risingwave_common::types::DataType;
344
345 use crate::catalog::root_catalog::SchemaPath;
346 use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
347
348 #[tokio::test]
349 async fn test_refresh_schema_rejects_associated_table() {
350 let proto_file = create_proto_file(PROTO_FILE_DATA);
351 let sql = format!(
352 r#"CREATE TABLE t
353 WITH (
354 connector = 'kafka',
355 topic = 'test-topic',
356 properties.bootstrap.server = 'localhost:29092'
357 )
358 FORMAT PLAIN ENCODE PROTOBUF (
359 message = '.test.TestRecord',
360 schema.location = 'file://{}'
361 )"#,
362 proto_file.path().to_str().unwrap()
363 );
364 let frontend = LocalFrontend::new(Default::default()).await;
365 frontend.run_sql(sql).await.unwrap();
366
367 let error = frontend
368 .run_sql("ALTER SOURCE t REFRESH SCHEMA")
369 .await
370 .unwrap_err();
371 assert!(error.to_string().contains("try to use ALTER TABLE instead"));
372 }
373
374 #[tokio::test]
375 async fn test_alter_source_with_sr_handler() {
376 let proto_file = create_proto_file(PROTO_FILE_DATA);
377 let sql = format!(
378 r#"CREATE SOURCE src
379 WITH (
380 connector = 'kafka',
381 topic = 'test-topic',
382 properties.bootstrap.server = 'localhost:29092'
383 )
384 FORMAT PLAIN ENCODE PROTOBUF (
385 message = '.test.TestRecord',
386 schema.location = 'file://{}'
387 )"#,
388 proto_file.path().to_str().unwrap()
389 );
390 let frontend = LocalFrontend::new(Default::default()).await;
391 let session = frontend.session_ref();
392 let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
393
394 frontend
395 .run_sql_with_session(session.clone(), "SET streaming_use_shared_source TO false;")
396 .await
397 .unwrap();
398 frontend
399 .run_sql_with_session(session.clone(), sql)
400 .await
401 .unwrap();
402
403 let get_source = || {
404 let catalog_reader = session.env().catalog_reader().read_guard();
405 catalog_reader
406 .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "src")
407 .unwrap()
408 .0
409 .clone()
410 };
411
412 let source = get_source();
413 expect_test::expect!["CREATE SOURCE src (id INT, country STRUCT<address CHARACTER VARYING, city STRUCT<address CHARACTER VARYING, zipcode CHARACTER VARYING>, zipcode CHARACTER VARYING>, zipcode BIGINT, rate REAL) WITH (connector = 'kafka', topic = 'test-topic', properties.bootstrap.server = 'localhost:29092') FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecord', schema.location = 'file://')"].assert_eq(&source.create_sql_purified().replace(proto_file.path().to_str().unwrap(), ""));
414
415 let sql = format!(
416 r#"ALTER SOURCE src FORMAT UPSERT ENCODE PROTOBUF (
417 message = '.test.TestRecord',
418 schema.location = 'file://{}'
419 )"#,
420 proto_file.path().to_str().unwrap()
421 );
422 assert!(
423 frontend
424 .run_sql(sql)
425 .await
426 .unwrap_err()
427 .to_string()
428 .contains("the original definition is FORMAT Plain ENCODE Protobuf")
429 );
430
431 let sql = format!(
432 r#"ALTER SOURCE src FORMAT PLAIN ENCODE PROTOBUF (
433 message = '.test.TestRecordAlterType',
434 schema.location = 'file://{}'
435 )"#,
436 proto_file.path().to_str().unwrap()
437 );
438 let res_str = frontend.run_sql(sql).await.unwrap_err().to_string();
439 assert!(res_str.contains("id: integer"));
440 assert!(res_str.contains("zipcode: bigint"));
441
442 let sql = format!(
443 r#"ALTER SOURCE src FORMAT PLAIN ENCODE PROTOBUF (
444 message = '.test.TestRecordExt',
445 schema.location = 'file://{}'
446 )"#,
447 proto_file.path().to_str().unwrap()
448 );
449 frontend.run_sql(sql).await.unwrap();
450
451 let altered_source = get_source();
452
453 let name_column = altered_source
454 .columns
455 .iter()
456 .find(|col| col.column_desc.name == "name")
457 .unwrap();
458 assert_eq!(name_column.column_desc.data_type, DataType::Varchar);
459
460 expect_test::expect!["CREATE SOURCE src (id INT, country STRUCT<address CHARACTER VARYING, city STRUCT<address CHARACTER VARYING, zipcode CHARACTER VARYING>, zipcode CHARACTER VARYING>, zipcode BIGINT, rate REAL, name CHARACTER VARYING) WITH (connector = 'kafka', topic = 'test-topic', properties.bootstrap.server = 'localhost:29092') FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecordExt', schema.location = 'file://')"].assert_eq(&altered_source.create_sql_purified().replace(proto_file.path().to_str().unwrap(), ""));
461 }
462}