Skip to main content

risingwave_frontend/handler/
alter_source_with_sr.rs

1// Copyright 2024 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16
17use either::Either;
18use itertools::Itertools;
19use pgwire::pg_response::StatementType;
20use risingwave_common::bail_not_implemented;
21use risingwave_common::catalog::{ColumnCatalog, max_column_id};
22use risingwave_connector::WithPropertiesExt;
23use risingwave_pb::catalog::StreamSourceInfo;
24use risingwave_pb::plan_common::{EncodeType, FormatType};
25use risingwave_sqlparser::ast::{
26    CompatibleFormatEncode, CreateSourceStatement, Encode, Format, FormatEncodeOptions, ObjectName,
27    SqlOption, Statement,
28};
29
30use super::create_source::{
31    generate_stream_graph_for_source, schema_has_schema_registry, validate_compatibility,
32};
33use super::util::SourceSchemaCompatExt;
34use super::{HandlerArgs, RwPgResponse};
35use crate::catalog::root_catalog::SchemaPath;
36use crate::catalog::source_catalog::SourceCatalog;
37use crate::error::{ErrorCode, Result};
38use crate::handler::create_source::{CreateSourceType, bind_columns_from_source};
39use crate::session::SessionImpl;
40use crate::utils::resolve_secret_ref_in_with_options;
41use crate::{Binder, WithOptions};
42
43fn format_type_to_format(from: FormatType) -> Option<Format> {
44    Some(match from {
45        FormatType::Unspecified => return None,
46        FormatType::Native => Format::Native,
47        FormatType::Debezium => Format::Debezium,
48        FormatType::DebeziumMongo => Format::DebeziumMongo,
49        FormatType::Maxwell => Format::Maxwell,
50        FormatType::Canal => Format::Canal,
51        FormatType::Upsert => Format::Upsert,
52        FormatType::Plain => Format::Plain,
53        FormatType::None => Format::None,
54    })
55}
56
57fn encode_type_to_encode(from: EncodeType) -> Option<Encode> {
58    Some(match from {
59        EncodeType::Unspecified => return None,
60        EncodeType::Native => Encode::Native,
61        EncodeType::Avro => Encode::Avro,
62        EncodeType::Csv => Encode::Csv,
63        EncodeType::Protobuf => Encode::Protobuf,
64        EncodeType::Json => Encode::Json,
65        EncodeType::Bytes => Encode::Bytes,
66        EncodeType::Template => Encode::Template,
67        EncodeType::Parquet => Encode::Parquet,
68        EncodeType::None => Encode::None,
69        EncodeType::Text => Encode::Text,
70    })
71}
72
73/// Returns the columns in `columns_a` but not in `columns_b`.
74///
75/// Note:
76/// - The comparison is done by name and data type, without checking `ColumnId`.
77/// - Hidden columns and `INCLUDE ... AS ...` columns are ignored. Because it's only for the special handling of alter sr.
78///   For the newly resolved `columns_from_resolve_source` (created by [`bind_columns_from_source`]), it doesn't contain hidden columns (`_row_id`) and `INCLUDE ... AS ...` columns.
79///   This is fragile and we should really refactor it later.
80/// - Generated columns are ignored when calculating dropped columns, because they are defined in SQL and should be preserved during schema refresh.
81/// - Column with the same name but different data type is considered as a different column, i.e., altering the data type of a column
82///   will be treated as dropping the old column and adding a new column. Note that we don't reject here like we do in `ALTER TABLE REFRESH SCHEMA`,
83///   because there's no data persistence (thus compatibility concern) in the source case.
84fn columns_minus(columns_a: &[ColumnCatalog], columns_b: &[ColumnCatalog]) -> Vec<ColumnCatalog> {
85    columns_a
86        .iter()
87        .filter(|col_a| {
88            !col_a.is_hidden()
89                && !col_a.is_connector_additional_column()
90                && !col_a.is_generated()
91                && !columns_b.iter().any(|col_b| {
92                    col_a.name() == col_b.name() && col_a.data_type() == col_b.data_type()
93                })
94        })
95        .cloned()
96        .collect()
97}
98
99/// Fetch the source catalog.
100pub fn fetch_source_catalog_with_db_schema_id(
101    session: &SessionImpl,
102    name: &ObjectName,
103) -> Result<Arc<SourceCatalog>> {
104    let db_name = &session.database();
105    let (schema_name, real_source_name) = Binder::resolve_schema_qualified_name(db_name, name)?;
106    let search_path = session.config().search_path();
107    let user_name = &session.user_name();
108
109    let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
110
111    let reader = session.env().catalog_reader().read_guard();
112    let (source, schema_name) =
113        reader.get_source_by_name(db_name, schema_path, &real_source_name)?;
114
115    session.check_privilege_for_drop_alter(schema_name, &**source)?;
116
117    Ok(Arc::clone(source))
118}
119
120/// Check if the original source is created with `FORMAT .. ENCODE ..` clause,
121/// and if the FORMAT and ENCODE are modified.
122pub fn check_format_encode(
123    original_source: &SourceCatalog,
124    new_format_encode: &FormatEncodeOptions,
125) -> Result<()> {
126    let StreamSourceInfo {
127        format, row_encode, ..
128    } = original_source.info;
129    let (Some(old_format), Some(old_row_encode)) = (
130        format_type_to_format(FormatType::try_from(format).unwrap()),
131        encode_type_to_encode(EncodeType::try_from(row_encode).unwrap()),
132    ) else {
133        return Err(ErrorCode::NotSupported(
134            "altering a legacy source which is not created using `FORMAT .. ENCODE ..` Clause"
135                .to_owned(),
136            "try this feature by creating a fresh source".to_owned(),
137        )
138        .into());
139    };
140
141    if new_format_encode.format != old_format || new_format_encode.row_encode != old_row_encode {
142        bail_not_implemented!(
143            "the original definition is FORMAT {:?} ENCODE {:?}, and altering them is not supported yet",
144            &old_format,
145            &old_row_encode,
146        );
147    }
148
149    Ok(())
150}
151
152/// Refresh the source registry and get the added/dropped columns.
153pub async fn refresh_sr_and_get_columns_diff(
154    original_source: &SourceCatalog,
155    format_encode: &FormatEncodeOptions,
156    session: &Arc<SessionImpl>,
157) -> Result<(StreamSourceInfo, Vec<ColumnCatalog>, Vec<ColumnCatalog>)> {
158    let mut with_properties = original_source.with_properties.clone();
159    let secret_refs = with_properties.as_secret().clone();
160    validate_compatibility(format_encode, &mut with_properties, &secret_refs)?;
161
162    if with_properties.is_cdc_connector() {
163        bail_not_implemented!("altering a cdc source is not supported");
164    }
165
166    let (Some(columns_from_resolve_source), source_info) = bind_columns_from_source(
167        session,
168        format_encode,
169        Either::Right(&with_properties),
170        CreateSourceType::for_replace(original_source),
171    )
172    .await?
173    else {
174        // Source without schema registry is rejected.
175        unreachable!("source without schema registry is rejected")
176    };
177
178    let mut added_columns = columns_minus(&columns_from_resolve_source, &original_source.columns);
179    // The newly resolved columns' column IDs also starts from 1. They cannot be used directly.
180    let mut next_col_id = max_column_id(&original_source.columns).next();
181    for col in &mut added_columns {
182        col.column_desc.column_id = next_col_id;
183        next_col_id = next_col_id.next();
184    }
185    let dropped_columns = columns_minus(&original_source.columns, &columns_from_resolve_source);
186    tracing::debug!(
187        ?added_columns,
188        ?dropped_columns,
189        ?columns_from_resolve_source,
190        original_source = ?original_source.columns
191    );
192
193    Ok((source_info, added_columns, dropped_columns))
194}
195
196fn get_format_encode_from_source(source: &SourceCatalog) -> Result<FormatEncodeOptions> {
197    let stmt = source.create_sql_ast()?;
198    let Statement::CreateSource {
199        stmt: CreateSourceStatement { format_encode, .. },
200    } = stmt
201    else {
202        return Err(ErrorCode::InternalError(format!(
203            "source \"{}\" has a non-CREATE SOURCE definition",
204            source.name
205        ))
206        .into());
207    };
208    Ok(format_encode.into_v2_with_warning())
209}
210
211fn reject_associated_table(source: &SourceCatalog) -> Result<()> {
212    if source.associated_table_id.is_some() {
213        return Err(ErrorCode::NotSupported(
214            "alter table with connector using ALTER SOURCE statement".to_owned(),
215            "try to use ALTER TABLE instead".to_owned(),
216        )
217        .into());
218    }
219
220    Ok(())
221}
222
223pub async fn handler_refresh_schema(
224    handler_args: HandlerArgs,
225    name: ObjectName,
226) -> Result<RwPgResponse> {
227    let source = fetch_source_catalog_with_db_schema_id(&handler_args.session, &name)?;
228    reject_associated_table(&source)?;
229    if source.is_cdc_table_source() {
230        return Err(ErrorCode::NotSupported(
231            "refreshing the schema of a CDC table source is not supported".to_owned(),
232            "Drop and recreate the CDC table source".to_owned(),
233        )
234        .into());
235    }
236    let format_encode = get_format_encode_from_source(&source)?;
237    handle_alter_source_with_sr(handler_args, name, format_encode).await
238}
239
240pub async fn handle_alter_source_with_sr(
241    handler_args: HandlerArgs,
242    name: ObjectName,
243    format_encode: FormatEncodeOptions,
244) -> Result<RwPgResponse> {
245    let session = handler_args.session.clone();
246    let source = fetch_source_catalog_with_db_schema_id(&session, &name)?;
247    let mut source = source.as_ref().clone();
248
249    reject_associated_table(&source)?;
250    if source.is_cdc_table_source() {
251        return Err(ErrorCode::NotSupported(
252            "altering the format or schema of a CDC table source is not supported".to_owned(),
253            "Drop and recreate the CDC table source".to_owned(),
254        )
255        .into());
256    }
257
258    check_format_encode(&source, &format_encode)?;
259
260    if !schema_has_schema_registry(&format_encode) {
261        return Err(ErrorCode::NotSupported(
262            "altering a source without schema registry".to_owned(),
263            "try `ALTER SOURCE .. ADD COLUMN ...` instead".to_owned(),
264        )
265        .into());
266    }
267
268    let (source_info, added_columns, dropped_columns) =
269        refresh_sr_and_get_columns_diff(&source, &format_encode, &session).await?;
270
271    if !dropped_columns.is_empty() {
272        bail_not_implemented!(
273            "this altering statement will drop columns, which is not supported yet: {}",
274            dropped_columns
275                .iter()
276                .map(|col| format!("({}: {})", col.name(), col.data_type()))
277                .join(", ")
278        );
279    }
280
281    source.info = source_info;
282    source.columns.extend(added_columns);
283    source.definition = alter_definition_format_encode(
284        source.create_sql_ast_purified()?,
285        format_encode.row_options.clone(),
286    )?;
287
288    let (format_encode_options, format_encode_secret_ref) = resolve_secret_ref_in_with_options(
289        WithOptions::try_from(format_encode.row_options())?,
290        session.as_ref(),
291    )?
292    .into_parts();
293    source
294        .info
295        .format_encode_options
296        .extend(format_encode_options);
297
298    source
299        .info
300        .format_encode_secret_refs
301        .extend(format_encode_secret_ref);
302
303    // update version
304    source.version += 1;
305
306    let pb_source = source.to_prost();
307    let catalog_writer = session.catalog_writer()?;
308    if source.info.is_shared() {
309        let graph = generate_stream_graph_for_source(handler_args, source.clone())?;
310        catalog_writer.replace_source(pb_source, graph).await?
311    } else {
312        catalog_writer.alter_source(pb_source).await?;
313    }
314    Ok(RwPgResponse::empty_result(StatementType::ALTER_SOURCE))
315}
316
317/// Apply the new `format_encode_options` to the source/table definition.
318pub fn alter_definition_format_encode(
319    mut stmt: Statement,
320    format_encode_options: Vec<SqlOption>,
321) -> Result<String> {
322    match &mut stmt {
323        Statement::CreateSource {
324            stmt: CreateSourceStatement { format_encode, .. },
325        }
326        | Statement::CreateTable {
327            format_encode: Some(format_encode),
328            ..
329        } => match format_encode {
330            CompatibleFormatEncode::V2(schema) => {
331                schema.row_options = format_encode_options;
332            }
333        },
334        _ => unreachable!(),
335    }
336
337    Ok(stmt.to_string())
338}
339
340#[cfg(test)]
341pub mod tests {
342    use risingwave_common::catalog::{DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME};
343    use risingwave_common::types::DataType;
344
345    use crate::catalog::root_catalog::SchemaPath;
346    use crate::test_utils::{LocalFrontend, PROTO_FILE_DATA, create_proto_file};
347
348    #[tokio::test]
349    async fn test_refresh_schema_rejects_associated_table() {
350        let proto_file = create_proto_file(PROTO_FILE_DATA);
351        let sql = format!(
352            r#"CREATE TABLE t
353            WITH (
354                connector = 'kafka',
355                topic = 'test-topic',
356                properties.bootstrap.server = 'localhost:29092'
357            )
358            FORMAT PLAIN ENCODE PROTOBUF (
359                message = '.test.TestRecord',
360                schema.location = 'file://{}'
361            )"#,
362            proto_file.path().to_str().unwrap()
363        );
364        let frontend = LocalFrontend::new(Default::default()).await;
365        frontend.run_sql(sql).await.unwrap();
366
367        let error = frontend
368            .run_sql("ALTER SOURCE t REFRESH SCHEMA")
369            .await
370            .unwrap_err();
371        assert!(error.to_string().contains("try to use ALTER TABLE instead"));
372    }
373
374    #[tokio::test]
375    async fn test_alter_source_with_sr_handler() {
376        let proto_file = create_proto_file(PROTO_FILE_DATA);
377        let sql = format!(
378            r#"CREATE SOURCE src
379            WITH (
380                connector = 'kafka',
381                topic = 'test-topic',
382                properties.bootstrap.server = 'localhost:29092'
383            )
384            FORMAT PLAIN ENCODE PROTOBUF (
385                message = '.test.TestRecord',
386                schema.location = 'file://{}'
387            )"#,
388            proto_file.path().to_str().unwrap()
389        );
390        let frontend = LocalFrontend::new(Default::default()).await;
391        let session = frontend.session_ref();
392        let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
393
394        frontend
395            .run_sql_with_session(session.clone(), "SET streaming_use_shared_source TO false;")
396            .await
397            .unwrap();
398        frontend
399            .run_sql_with_session(session.clone(), sql)
400            .await
401            .unwrap();
402
403        let get_source = || {
404            let catalog_reader = session.env().catalog_reader().read_guard();
405            catalog_reader
406                .get_source_by_name(DEFAULT_DATABASE_NAME, schema_path, "src")
407                .unwrap()
408                .0
409                .clone()
410        };
411
412        let source = get_source();
413        expect_test::expect!["CREATE SOURCE src (id INT, country STRUCT<address CHARACTER VARYING, city STRUCT<address CHARACTER VARYING, zipcode CHARACTER VARYING>, zipcode CHARACTER VARYING>, zipcode BIGINT, rate REAL) WITH (connector = 'kafka', topic = 'test-topic', properties.bootstrap.server = 'localhost:29092') FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecord', schema.location = 'file://')"].assert_eq(&source.create_sql_purified().replace(proto_file.path().to_str().unwrap(), ""));
414
415        let sql = format!(
416            r#"ALTER SOURCE src FORMAT UPSERT ENCODE PROTOBUF (
417                message = '.test.TestRecord',
418                schema.location = 'file://{}'
419            )"#,
420            proto_file.path().to_str().unwrap()
421        );
422        assert!(
423            frontend
424                .run_sql(sql)
425                .await
426                .unwrap_err()
427                .to_string()
428                .contains("the original definition is FORMAT Plain ENCODE Protobuf")
429        );
430
431        let sql = format!(
432            r#"ALTER SOURCE src FORMAT PLAIN ENCODE PROTOBUF (
433                message = '.test.TestRecordAlterType',
434                schema.location = 'file://{}'
435            )"#,
436            proto_file.path().to_str().unwrap()
437        );
438        let res_str = frontend.run_sql(sql).await.unwrap_err().to_string();
439        assert!(res_str.contains("id: integer"));
440        assert!(res_str.contains("zipcode: bigint"));
441
442        let sql = format!(
443            r#"ALTER SOURCE src FORMAT PLAIN ENCODE PROTOBUF (
444                message = '.test.TestRecordExt',
445                schema.location = 'file://{}'
446            )"#,
447            proto_file.path().to_str().unwrap()
448        );
449        frontend.run_sql(sql).await.unwrap();
450
451        let altered_source = get_source();
452
453        let name_column = altered_source
454            .columns
455            .iter()
456            .find(|col| col.column_desc.name == "name")
457            .unwrap();
458        assert_eq!(name_column.column_desc.data_type, DataType::Varchar);
459
460        expect_test::expect!["CREATE SOURCE src (id INT, country STRUCT<address CHARACTER VARYING, city STRUCT<address CHARACTER VARYING, zipcode CHARACTER VARYING>, zipcode CHARACTER VARYING>, zipcode BIGINT, rate REAL, name CHARACTER VARYING) WITH (connector = 'kafka', topic = 'test-topic', properties.bootstrap.server = 'localhost:29092') FORMAT PLAIN ENCODE PROTOBUF (message = '.test.TestRecordExt', schema.location = 'file://')"].assert_eq(&altered_source.create_sql_purified().replace(proto_file.path().to_str().unwrap(), ""));
461    }
462}