Skip to main content

risingwave_frontend/handler/create_source/
validate.rs

1// Copyright 2024 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use risingwave_connector::error::ConnectorError;
16use risingwave_connector::source::{ADBC_SNOWFLAKE_CONNECTOR, BATCH_POSIX_FS_CONNECTOR};
17use risingwave_pb::secret::PbSecretRef;
18
19use super::*;
20
21pub static SOURCE_ALLOWED_CONNECTION_CONNECTOR: LazyLock<HashSet<PbConnectionType>> =
22    LazyLock::new(|| {
23        hashset! {
24            PbConnectionType::Unspecified,
25            PbConnectionType::Kafka,
26            PbConnectionType::Iceberg,
27        }
28    });
29
30pub static SOURCE_ALLOWED_CONNECTION_SCHEMA_REGISTRY: LazyLock<HashSet<PbConnectionType>> =
31    LazyLock::new(|| {
32        hashset! {
33            PbConnectionType::Unspecified,
34            PbConnectionType::SchemaRegistry,
35        }
36    });
37
38// TODO: Better design if we want to support ENCODE KEY where we will have 4 dimensional array
39static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, Vec<Encode>>>> =
40    LazyLock::new(|| {
41        convert_args!(hashmap!(
42                KAFKA_CONNECTOR => hashmap!(
43                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes, Encode::Csv],
44                    Format::Upsert => vec![Encode::Json, Encode::Avro, Encode::Protobuf],
45                    Format::Debezium => vec![Encode::Json, Encode::Avro],
46                    Format::DebeziumMongo => vec![Encode::Json],
47                ),
48                PULSAR_CONNECTOR => hashmap!(
49                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes],
50                    Format::Upsert => vec![Encode::Json, Encode::Avro],
51                    Format::Debezium => vec![Encode::Json],
52                ),
53                KINESIS_CONNECTOR => hashmap!(
54                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes, Encode::Csv],
55                    Format::Upsert => vec![Encode::Json, Encode::Avro],
56                    Format::Debezium => vec![Encode::Json],
57                ),
58                GOOGLE_PUBSUB_CONNECTOR => hashmap!(
59                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes],
60                    Format::Debezium => vec![Encode::Json],
61                ),
62                NEXMARK_CONNECTOR => hashmap!(
63                    Format::Native => vec![Encode::Native],
64                    Format::Plain => vec![Encode::Bytes],
65                ),
66                DATAGEN_CONNECTOR => hashmap!(
67                    Format::Native => vec![Encode::Native],
68                    Format::Plain => vec![Encode::Bytes, Encode::Json],
69                ),
70                OPENDAL_S3_CONNECTOR => hashmap!(
71                    Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
72                ),
73                GCS_CONNECTOR => hashmap!(
74                    Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
75                ),
76                AZBLOB_CONNECTOR => hashmap!(
77                    Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
78                ),
79                POSIX_FS_CONNECTOR => hashmap!(
80                    Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
81                ),
82                BATCH_POSIX_FS_CONNECTOR => hashmap!(
83                    Format::Plain => vec![Encode::Csv],
84                ),
85                NATS_CONNECTOR => hashmap!(
86                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Bytes],
87                ),
88                MQTT_CONNECTOR => hashmap!(
89                    Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Bytes],
90                ),
91                TEST_CONNECTOR => hashmap!(
92                    Format::Plain => vec![Encode::Json],
93                ),
94                ICEBERG_CONNECTOR => hashmap!(
95                    Format::None => vec![Encode::None],
96                ),
97                ADBC_SNOWFLAKE_CONNECTOR => hashmap!(
98                    Format::None => vec![Encode::None],
99                ),
100        ))
101    });
102
103/// Returns whether the Pulsar schema is used. The Pulsar schema options themselves are validated by
104/// `PulsarSchemaConfig::from_options` when building the parser config. Only the connector and
105/// format, which that config is unaware of, are checked here.
106fn validate_pulsar_schema_options(
107    format_encode: &FormatEncodeOptions,
108    connector: &str,
109) -> Result<bool> {
110    let options = WithOptions::try_from(format_encode.row_options())?;
111    if !uses_pulsar_schema(&options, options.secret_ref()) {
112        return Ok(false);
113    }
114
115    if connector != PULSAR_CONNECTOR
116        || format_encode.format != Format::Plain
117        || !matches!(format_encode.row_encode, Encode::Avro | Encode::Json)
118    {
119        return Err(RwError::from(ProtocolError(format!(
120            "Pulsar schema requires connector = '{PULSAR_CONNECTOR}' with FORMAT PLAIN ENCODE AVRO or JSON"
121        ))));
122    }
123
124    Ok(true)
125}
126
127pub fn validate_compatibility(
128    format_encode: &FormatEncodeOptions,
129    props: &mut BTreeMap<String, String>,
130    secret_refs: &BTreeMap<String, PbSecretRef>,
131) -> Result<()> {
132    let mut connector = props
133        .get_connector()
134        .ok_or_else(|| RwError::from(ProtocolError("missing field 'connector'".to_owned())))?;
135    let uses_pulsar_schema = validate_pulsar_schema_options(format_encode, &connector)?;
136
137    let protocol_error = |e: ConnectorError| RwError::from(ProtocolError(e.to_report_string()));
138    dispatch_cdc_source_type!(
139        props.cdc_source_type(),
140        T,
141        {
142            T::check_license()?;
143            T::check_format_encode(
144                format_to_prost(&format_encode.format),
145                row_encode_to_prost(&format_encode.row_encode),
146            )
147            .map_err(protocol_error)?;
148            T::check_and_fill_create_with_options(props, secret_refs).map_err(protocol_error)?;
149            return Ok(());
150        },
151        {}
152    );
153
154    if connector == OPENDAL_S3_CONNECTOR {
155        // reject s3_v2 creation
156        return Err(RwError::from(Deprecated(
157            OPENDAL_S3_CONNECTOR.to_owned(),
158            LEGACY_S3_CONNECTOR.to_owned(),
159        )));
160    }
161    if connector == LEGACY_S3_CONNECTOR {
162        // S3 connector is deprecated, use OPENDAL_S3_CONNECTOR instead
163        // do s3 -> s3_v2 migration
164        let entry = props.get_mut(UPSTREAM_SOURCE_KEY).unwrap();
165        *entry = OPENDAL_S3_CONNECTOR.to_owned();
166        connector = OPENDAL_S3_CONNECTOR.to_owned();
167    }
168
169    let compatible_formats = CONNECTORS_COMPATIBLE_FORMATS
170        .get(&connector)
171        .ok_or_else(|| {
172            RwError::from(ProtocolError(format!(
173                "connector {:?} is not supported, accept {:?}",
174                connector,
175                CONNECTORS_COMPATIBLE_FORMATS.keys()
176            )))
177        })?;
178    if connector != KAFKA_CONNECTOR && !uses_pulsar_schema {
179        let res = match (&format_encode.format, &format_encode.row_encode) {
180            (Format::Plain | Format::Upsert, Encode::Protobuf | Encode::Avro) => {
181                let mut options = WithOptions::try_from(format_encode.row_options())?;
182                let (_, use_schema_registry) =
183                    get_schema_location(options.inner_mut()).unwrap_or_default();
184                use_schema_registry
185            }
186            (Format::Debezium, Encode::Avro) => true,
187            (_, _) => false,
188        };
189        if res {
190            return Err(RwError::from(ProtocolError(format!(
191                "The {} must be kafka when schema registry is used",
192                UPSTREAM_SOURCE_KEY
193            ))));
194        }
195    }
196
197    let compatible_encodes = compatible_formats
198        .get(&format_encode.format)
199        .ok_or_else(|| {
200            RwError::from(ProtocolError(format!(
201                "connector {} does not support format {:?}",
202                connector, format_encode.format
203            )))
204        })?;
205    if !compatible_encodes.contains(&format_encode.row_encode) {
206        return Err(RwError::from(ProtocolError(format!(
207            "connector {} does not support format {:?} with encode {:?}",
208            connector, format_encode.format, format_encode.row_encode
209        ))));
210    }
211    Ok(())
212}
213
214#[cfg(test)]
215mod tests {
216    use risingwave_sqlparser::ast::SqlOption;
217
218    use super::*;
219
220    fn format_encode(
221        format: Format,
222        encode: Encode,
223        options: &[(&str, &str)],
224    ) -> FormatEncodeOptions {
225        let row_options = options
226            .iter()
227            .map(|(name, value)| {
228                let name = name.to_string();
229                let value = value.to_string();
230                SqlOption::try_from((&name, &value)).unwrap()
231            })
232            .collect();
233        FormatEncodeOptions {
234            format,
235            row_encode: encode,
236            row_options,
237            key_encode: None,
238        }
239    }
240
241    fn source_options(connector: &str) -> BTreeMap<String, String> {
242        BTreeMap::from([(UPSTREAM_SOURCE_KEY.to_owned(), connector.to_owned())])
243    }
244
245    #[test]
246    fn pulsar_schema_accepts_minimal_plain_avro_and_json_options() {
247        for encode in [Encode::Avro, Encode::Json] {
248            let format_encode = format_encode(
249                Format::Plain,
250                encode,
251                &[
252                    (PULSAR_SCHEMA_URL_KEY, "https://localhost:8443"),
253                    (PULSAR_SCHEMA_AUTH_TOKEN_KEY, "schema-token"),
254                ],
255            );
256            validate_compatibility(
257                &format_encode,
258                &mut source_options(PULSAR_CONNECTOR),
259                &BTreeMap::new(),
260            )
261            .unwrap();
262        }
263    }
264
265    #[test]
266    fn pulsar_schema_rejects_other_connectors_and_formats() {
267        for (connector, format, encode) in [
268            (KAFKA_CONNECTOR, Format::Plain, Encode::Avro),
269            (PULSAR_CONNECTOR, Format::Upsert, Encode::Avro),
270            (KAFKA_CONNECTOR, Format::Plain, Encode::Json),
271            (PULSAR_CONNECTOR, Format::Upsert, Encode::Json),
272            (PULSAR_CONNECTOR, Format::Plain, Encode::Protobuf),
273        ] {
274            let format_encode = format_encode(
275                format,
276                encode,
277                &[(PULSAR_SCHEMA_URL_KEY, "http://localhost:8080")],
278            );
279            let error = validate_compatibility(
280                &format_encode,
281                &mut source_options(connector),
282                &BTreeMap::new(),
283            )
284            .unwrap_err();
285            assert!(error.to_string().contains(
286                "Pulsar schema requires connector = 'pulsar' with FORMAT PLAIN ENCODE AVRO or JSON"
287            ));
288        }
289    }
290
291    #[test]
292    fn confluent_schema_registry_behavior_is_unchanged() {
293        let format_encode = format_encode(
294            Format::Plain,
295            Encode::Avro,
296            &[("schema.registry", "http://localhost:8081")],
297        );
298        validate_compatibility(
299            &format_encode,
300            &mut source_options(KAFKA_CONNECTOR),
301            &BTreeMap::new(),
302        )
303        .unwrap();
304    }
305
306    #[test]
307    fn glue_schema_registry_non_kafka() {
308        let format_encode = format_encode(
309            Format::Plain,
310            Encode::Avro,
311            &[(AWS_GLUE_SCHEMA_ARN_KEY, "placeholder")],
312        );
313        validate_compatibility(
314            &format_encode,
315            &mut source_options(KINESIS_CONNECTOR),
316            &BTreeMap::new(),
317        )
318        .unwrap();
319    }
320}