Skip to main content

risingwave_frontend/handler/create_source/
external_schema.rs

1// Copyright 2024 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! bind columns from external schema
16
17use risingwave_connector::source::ADBC_SNOWFLAKE_CONNECTOR;
18
19use super::*;
20
21mod json;
22use json::*;
23mod avro;
24use avro::extract_avro_table_schema;
25pub mod debezium;
26pub mod iceberg;
27use iceberg::extract_iceberg_columns;
28mod protobuf;
29use protobuf::extract_protobuf_table_schema;
30pub mod nexmark;
31
32/// Define a feature-gated function from a module.
33///
34/// This macro conditionally includes a module and re-exports a function from it.
35/// When the feature is disabled, it generates a stub function that returns an error.
36macro_rules! feature_gated_function {
37    (
38        mod $mod_name:ident,
39        $feature_name:literal,
40        async fn $func_name:ident ( $( $param_name:ident : $param_type:ty ),* $(,)? ) -> $ret_type:ty
41    ) => {
42        #[cfg(feature = $feature_name)]
43        pub mod $mod_name;
44        #[cfg(feature = $feature_name)]
45        use $mod_name::$func_name;
46
47        #[cfg(not(feature = $feature_name))]
48        #[allow(unused_variables)]
49        pub async fn $func_name( $( $param_name : $param_type ),* ) -> $ret_type {
50            Err(anyhow::anyhow!(
51                "Feature `{}` is not enabled at compile time. \
52                Please enable it in `Cargo.toml` and rebuild.",
53                $feature_name
54            ))
55        }
56    };
57}
58
59feature_gated_function!(
60    mod adbc_snowflake,
61    "source-adbc_snowflake",
62    async fn extract_adbc_snowflake_columns(
63        with_properties: &WithOptionsSecResolved,
64    ) -> anyhow::Result<Vec<ColumnCatalog>>
65);
66
67/// Resolves the schema of the source from external schema file.
68/// See <https://docs.risingwave.com/sql/commands/sql-create-source> for more information.
69///
70/// Note: the returned schema strictly corresponds to the schema.
71/// Other special columns like additional columns (`INCLUDE`), and `row_id` column are not included.
72pub async fn bind_columns_from_source(
73    session: &SessionImpl,
74    format_encode: &FormatEncodeOptions,
75    with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
76    create_source_type: CreateSourceType,
77) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
78    let (columns_from_resolve_source, mut source_info) =
79        if create_source_type == CreateSourceType::SharedCdc {
80            bind_columns_from_source_for_cdc(session, format_encode)?
81        } else {
82            bind_columns_from_source_for_non_cdc(session, format_encode, with_properties).await?
83        };
84    if create_source_type.is_shared() {
85        // Note: this field should be called is_shared. Check field doc for more details.
86        source_info.cdc_source_job = true;
87        source_info.is_distributed = create_source_type == CreateSourceType::SharedNonCdc;
88    }
89    Ok((columns_from_resolve_source, source_info))
90}
91
92async fn bind_columns_from_source_for_non_cdc(
93    session: &SessionImpl,
94    format_encode: &FormatEncodeOptions,
95    with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
96) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
97    const MESSAGE_NAME_KEY: &str = "message";
98    const KEY_MESSAGE_NAME_KEY: &str = "key.message";
99    const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy";
100
101    let options_with_secret = match with_properties {
102        Either::Left(options) => {
103            let (sec_resolve_props, connection_type, _) = resolve_connection_ref_and_secret_ref(
104                options.clone(),
105                session,
106                Some(TelemetryDatabaseObject::Source),
107            )?;
108            if !SOURCE_ALLOWED_CONNECTION_CONNECTOR.contains(&connection_type) {
109                return Err(RwError::from(ProtocolError(format!(
110                    "connection type {:?} is not allowed, allowed types: {:?}",
111                    connection_type, SOURCE_ALLOWED_CONNECTION_CONNECTOR
112                ))));
113            }
114
115            sec_resolve_props
116        }
117        Either::Right(options_with_secret) => options_with_secret.clone(),
118    };
119
120    let is_kafka: bool = options_with_secret.is_kafka_connector();
121
122    // todo: need to resolve connection ref for schema registry
123    let (sec_resolve_props, connection_type, schema_registry_conn_ref) =
124        resolve_connection_ref_and_secret_ref(
125            WithOptions::try_from(format_encode.row_options())?,
126            session,
127            Some(TelemetryDatabaseObject::Source),
128        )?;
129    ensure_connection_type_allowed(connection_type, &SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY)?;
130
131    let (format_encode_options, format_encode_secret_refs) = sec_resolve_props.into_parts();
132    // Need real secret to access the schema registry
133    let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
134        format_encode_options.clone(),
135        format_encode_secret_refs.clone(),
136    )?;
137
138    fn get_key_message_name(options: &mut BTreeMap<String, String>) -> Option<String> {
139        consume_string_from_options(options, KEY_MESSAGE_NAME_KEY).ok()
140    }
141    fn get_sr_name_strategy_check(
142        options: &mut BTreeMap<String, String>,
143        use_sr: bool,
144    ) -> Result<Option<i32>> {
145        let name_strategy = get_name_strategy_or_default(try_consume_string_from_options(
146            options,
147            NAME_STRATEGY_KEY,
148        ))?;
149        if !use_sr && name_strategy.is_some() {
150            return Err(RwError::from(ProtocolError(
151                "schema registry name strategy only works with schema registry enabled".to_owned(),
152            )));
153        }
154        Ok(name_strategy)
155    }
156
157    let mut stream_source_info = StreamSourceInfo {
158        format: format_to_prost(&format_encode.format) as i32,
159        row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
160        format_encode_options,
161        format_encode_secret_refs,
162        connection_id: schema_registry_conn_ref,
163        ..Default::default()
164    };
165
166    if format_encode.format == Format::Debezium {
167        try_consume_string_from_options(&mut format_encode_options_to_consume, DEBEZIUM_IGNORE_KEY);
168    }
169
170    let columns = match (&format_encode.format, &format_encode.row_encode) {
171        (Format::Native, Encode::Native)
172        | (Format::Plain, Encode::Bytes)
173        | (Format::DebeziumMongo, Encode::Json) => None,
174        (Format::Plain, Encode::Protobuf) | (Format::Upsert, Encode::Protobuf) => {
175            let (row_schema_location, use_schema_registry) =
176                get_schema_location(&mut format_encode_options_to_consume)?;
177            let message_name = consume_string_from_options(
178                &mut format_encode_options_to_consume,
179                MESSAGE_NAME_KEY,
180            )?;
181            let name_strategy = get_sr_name_strategy_check(
182                &mut format_encode_options_to_consume,
183                use_schema_registry,
184            )?;
185
186            stream_source_info.use_schema_registry = use_schema_registry;
187            stream_source_info
188                .row_schema_location
189                .clone_from(&row_schema_location);
190            stream_source_info
191                .proto_message_name
192                .clone_from(&message_name);
193            stream_source_info.key_message_name =
194                get_key_message_name(&mut format_encode_options_to_consume);
195            stream_source_info.name_strategy =
196                name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
197
198            Some(
199                extract_protobuf_table_schema(
200                    &stream_source_info,
201                    &options_with_secret,
202                    &mut format_encode_options_to_consume,
203                )
204                .await?,
205            )
206        }
207        (format @ (Format::Plain | Format::Upsert | Format::Debezium), Encode::Avro) => {
208            if format_encode_options_to_consume.contains_key(PULSAR_SCHEMA_URL_KEY) {
209                // Pulsar schema options are parsed by `SpecificParserConfig` and remain in the
210                // catalog format options. They deliberately do not populate the legacy
211                // Confluent-oriented `row_schema_location` fields.
212            } else if format_encode_options_to_consume
213                .remove(AWS_GLUE_SCHEMA_ARN_KEY)
214                .is_none()
215            {
216                // Legacy logic that assumes either `schema.location` or confluent `schema.registry`.
217                // The handling of newly added aws glue is centralized in `connector::parser`.
218                // TODO(xiangjinwu): move these option parsing to `connector::parser` as well,
219                // together with schema registry logic in `validate::validate_compatibility`
220
221                let (row_schema_location, use_schema_registry) =
222                    get_schema_location(&mut format_encode_options_to_consume)?;
223
224                if matches!(format, Format::Debezium) && !use_schema_registry {
225                    return Err(RwError::from(ProtocolError(
226                        "schema location for DEBEZIUM_AVRO row format is not supported".to_owned(),
227                    )));
228                }
229
230                let message_name = try_consume_string_from_options(
231                    &mut format_encode_options_to_consume,
232                    MESSAGE_NAME_KEY,
233                );
234                let name_strategy = get_sr_name_strategy_check(
235                    &mut format_encode_options_to_consume,
236                    use_schema_registry,
237                )?;
238
239                stream_source_info.use_schema_registry = use_schema_registry;
240                stream_source_info
241                    .row_schema_location
242                    .clone_from(&row_schema_location);
243                stream_source_info.proto_message_name = message_name.unwrap_or_default();
244                stream_source_info.key_message_name =
245                    get_key_message_name(&mut format_encode_options_to_consume);
246                stream_source_info.name_strategy =
247                    name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
248            }
249
250            Some(
251                extract_avro_table_schema(
252                    &stream_source_info,
253                    &options_with_secret,
254                    &mut format_encode_options_to_consume,
255                    matches!(format, Format::Debezium),
256                )
257                .await?,
258            )
259        }
260        (Format::Plain, Encode::Csv) => {
261            let chars =
262                consume_string_from_options(&mut format_encode_options_to_consume, "delimiter")?;
263            let delimiter = get_delimiter(chars.as_str()).context("failed to parse delimiter")?;
264            let has_header = try_consume_string_from_options(
265                &mut format_encode_options_to_consume,
266                "without_header",
267            )
268            .map(|s| s == "false")
269            .unwrap_or(true);
270
271            if is_kafka && has_header {
272                return Err(RwError::from(ProtocolError(
273                    "CSV HEADER is not supported when creating table with Kafka connector"
274                        .to_owned(),
275                )));
276            }
277
278            stream_source_info.csv_delimiter = delimiter as i32;
279            stream_source_info.csv_has_header = has_header;
280
281            None
282        }
283        // For parquet format, this step is implemented in parquet parser.
284        (Format::Plain, Encode::Parquet) => None,
285        (
286            Format::Plain | Format::Upsert | Format::Maxwell | Format::Canal | Format::Debezium,
287            Encode::Json,
288        ) => {
289            if matches!(
290                format_encode.format,
291                Format::Plain | Format::Upsert | Format::Debezium
292            ) {
293                // Parse the value but throw it away.
294                // It would be too late to report error in `SpecificParserConfig::new`,
295                // which leads to recovery loop.
296                // TODO: rely on SpecificParserConfig::new to validate, like Avro
297                if let Some(value) =
298                    format_encode_options_to_consume.get(TimestamptzHandling::OPTION_KEY)
299                {
300                    TimestamptzHandling::from_options(value)
301                        .map_err(|err| InvalidInputSyntax(err.message))?;
302                }
303                try_consume_string_from_options(
304                    &mut format_encode_options_to_consume,
305                    TimestamptzHandling::OPTION_KEY,
306                );
307            }
308
309            if let Some(config) =
310                PulsarSchemaConfig::from_options(&format_encode_options_to_consume)?
311            {
312                Some(
313                    extract_pulsar_json_table_schema(
314                        &config,
315                        &options_with_secret,
316                        &mut format_encode_options_to_consume,
317                    )
318                    .await?,
319                )
320            } else {
321                let schema_config =
322                    get_json_schema_location(&mut format_encode_options_to_consume)?;
323                stream_source_info.use_schema_registry =
324                    json_schema_infer_use_schema_registry(&schema_config);
325
326                extract_json_table_schema(
327                    &schema_config,
328                    &options_with_secret,
329                    &mut format_encode_options_to_consume,
330                )
331                .await?
332            }
333        }
334        (Format::None, Encode::None) => {
335            if options_with_secret.is_iceberg_connector() {
336                Some(
337                    extract_iceberg_columns(&options_with_secret)
338                        .await
339                        .map_err(|err| ProtocolError(err.to_report_string()))?,
340                )
341            } else if options_with_secret
342                .get(UPSTREAM_SOURCE_KEY)
343                .is_some_and(|s| s.eq_ignore_ascii_case(ADBC_SNOWFLAKE_CONNECTOR))
344            {
345                Some(
346                    extract_adbc_snowflake_columns(&options_with_secret)
347                        .await
348                        .map_err(|err| ProtocolError(err.to_report_string()))?,
349                )
350            } else {
351                None
352            }
353        }
354        (format, encoding) => {
355            return Err(RwError::from(ProtocolError(format!(
356                "Unknown combination {:?} {:?}",
357                format, encoding
358            ))));
359        }
360    };
361
362    if !format_encode_options_to_consume.is_empty() {
363        let err_string = format!(
364            "Get unknown format_encode_options for {:?} {:?}: {}",
365            format_encode.format,
366            format_encode.row_encode,
367            format_encode_options_to_consume
368                .keys()
369                .cloned()
370                .collect::<Vec<String>>()
371                .join(","),
372        );
373        session.notice_to_user(err_string);
374    }
375    Ok((columns, stream_source_info))
376}
377
378fn bind_columns_from_source_for_cdc(
379    session: &SessionImpl,
380    format_encode: &FormatEncodeOptions,
381) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
382    let with_options = WithOptions::try_from(format_encode.row_options())?;
383    if !with_options.connection_ref().is_empty() {
384        return Err(RwError::from(NotSupported(
385            "CDC connector does not support connection ref yet".to_owned(),
386            "Explicitly specify the connection in WITH clause".to_owned(),
387        )));
388    }
389    let (format_encode_options, format_encode_secret_refs) =
390        resolve_secret_ref_in_with_options(with_options, session)?.into_parts();
391
392    // Need real secret to access the schema registry
393    let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
394        format_encode_options.clone(),
395        format_encode_secret_refs.clone(),
396    )?;
397
398    match (&format_encode.format, &format_encode.row_encode) {
399        (Format::Plain, Encode::Json) => (),
400        (format, encoding) => {
401            // Note: parser will also check this. Just be extra safe here
402            return Err(RwError::from(ProtocolError(format!(
403                "Row format for CDC connectors should be either omitted or set to `FORMAT PLAIN ENCODE JSON`, got: {:?} {:?}",
404                format, encoding
405            ))));
406        }
407    };
408
409    let columns = ColumnCatalog::debezium_cdc_source_cols().to_vec();
410    let schema_config = get_json_schema_location(&mut format_encode_options_to_consume)?;
411
412    let stream_source_info = StreamSourceInfo {
413        format: format_to_prost(&format_encode.format) as i32,
414        row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
415        format_encode_options,
416        use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
417        cdc_source_job: true,
418        is_distributed: false,
419        format_encode_secret_refs,
420        ..Default::default()
421    };
422    if !format_encode_options_to_consume.is_empty() {
423        let err_string = format!(
424            "Get unknown format_encode_options for {:?} {:?}: {}",
425            format_encode.format,
426            format_encode.row_encode,
427            format_encode_options_to_consume
428                .keys()
429                .cloned()
430                .collect::<Vec<String>>()
431                .join(","),
432        );
433        session.notice_to_user(err_string);
434    }
435    Ok((Some(columns), stream_source_info))
436}
437
438pub(super) fn format_to_prost(format: &Format) -> FormatType {
439    match format {
440        Format::Native => FormatType::Native,
441        Format::Plain => FormatType::Plain,
442        Format::Upsert => FormatType::Upsert,
443        Format::Debezium => FormatType::Debezium,
444        Format::DebeziumMongo => FormatType::DebeziumMongo,
445        Format::Maxwell => FormatType::Maxwell,
446        Format::Canal => FormatType::Canal,
447        Format::None => FormatType::None,
448    }
449}
450pub(super) fn row_encode_to_prost(row_encode: &Encode) -> EncodeType {
451    match row_encode {
452        Encode::Native => EncodeType::Native,
453        Encode::Json => EncodeType::Json,
454        Encode::Avro => EncodeType::Avro,
455        Encode::Protobuf => EncodeType::Protobuf,
456        Encode::Csv => EncodeType::Csv,
457        Encode::Bytes => EncodeType::Bytes,
458        Encode::Template => EncodeType::Template,
459        Encode::Parquet => EncodeType::Parquet,
460        Encode::None => EncodeType::None,
461        Encode::Text => EncodeType::Text,
462    }
463}
464
465pub fn get_schema_location(
466    format_encode_options: &mut BTreeMap<String, String>,
467) -> Result<(String, bool)> {
468    let schema_location = try_consume_string_from_options(format_encode_options, "schema.location");
469    let schema_registry = try_consume_string_from_options(format_encode_options, "schema.registry");
470    match (schema_location, schema_registry) {
471        (None, None) => Err(RwError::from(ProtocolError(
472            "missing either a schema location or a schema registry".to_owned(),
473        ))),
474        (None, Some(schema_registry)) => Ok((schema_registry, true)),
475        (Some(schema_location), None) => Ok((schema_location, false)),
476        (Some(_), Some(_)) => Err(RwError::from(ProtocolError(
477            "only need either the schema location or the schema registry".to_owned(),
478        ))),
479    }
480}
481
482pub fn schema_has_schema_registry(schema: &FormatEncodeOptions) -> bool {
483    match schema.row_encode {
484        Encode::Avro | Encode::Protobuf => true,
485        Encode::Json => {
486            let mut options = WithOptions::try_from(schema.row_options()).unwrap();
487            uses_pulsar_schema(&options, options.secret_ref())
488                || matches!(get_json_schema_location(options.inner_mut()), Ok(Some(_)))
489        }
490        _ => false,
491    }
492}
493
494#[inline]
495fn get_name_strategy_or_default(name_strategy: Option<String>) -> Result<Option<i32>> {
496    match name_strategy {
497        None => Ok(None),
498        Some(name) => Ok(Some(name_strategy_from_str(name.as_str())
499            .ok_or_else(|| RwError::from(ProtocolError(format!("\
500            expect strategy name in topic_name_strategy, record_name_strategy and topic_record_name_strategy, but got {}", name))))? as i32)),
501    }
502}