Skip to main content

risingwave_frontend/optimizer/plan_node/
stream_cdc_table_scan.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use itertools::Itertools;
16use pretty_xmlish::{Pretty, XmlNode};
17use risingwave_common::catalog::{ColumnCatalog, Field};
18use risingwave_common::types::DataType;
19use risingwave_common::util::sort_util::OrderType;
20use risingwave_connector::source::cdc::CdcSourceExternalTableIdentifier;
21use risingwave_connector::{WithPropertiesExt, dispatch_cdc_source_type};
22use risingwave_pb::stream_plan::PbStreamNode;
23use risingwave_pb::stream_plan::stream_node::{PbNodeBody, PbStreamKind};
24
25use super::stream::prelude::*;
26use super::utils::{Distill, childless_record};
27use super::{ExprRewritable, PlanBase, StreamNode, StreamPlanRef as PlanRef, generic};
28use crate::catalog::ColumnId;
29use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprType, ExprVisitor, FunctionCall, InputRef};
30use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
31use crate::optimizer::plan_node::utils::{IndicesDisplay, TableCatalogBuilder};
32use crate::optimizer::property::{Distribution, DistributionDisplay};
33use crate::scheduler::SchedulerResult;
34use crate::stream_fragmenter::BuildFragmentGraphState;
35use crate::{Explain, TableCatalog};
36
37/// `StreamCdcTableScan` is a virtual plan node to represent a stream cdc table scan.
38/// It will be converted to cdc backfill + merge node (for upstream source)
39#[derive(Debug, Clone, PartialEq, Eq, Hash)]
40pub struct StreamCdcTableScan {
41    pub base: PlanBase<Stream>,
42    core: generic::CdcScan,
43}
44
45impl StreamCdcTableScan {
46    pub fn new(core: generic::CdcScan) -> Self {
47        let distribution = Distribution::SomeShard;
48        let base = PlanBase::new_stream_with_core(
49            &core,
50            distribution,
51            // Debezium table scans emit upsert semantics: updates contain only the new value,
52            // and deletes may contain only primary-key columns. Declaring this correctly lets
53            // downstream materialization add conflict handling before stateful consumption.
54            StreamKind::Upsert,
55            false,
56            core.watermark_columns(),
57            core.columns_monotonicity(),
58        );
59        Self { base, core }
60    }
61
62    pub fn table_name(&self) -> &str {
63        &self.core.table_name
64    }
65
66    pub fn core(&self) -> &generic::CdcScan {
67        &self.core
68    }
69
70    /// Build catalog for cdc backfill state.
71    ///
72    /// For non-parallelized cdc backfill:
73    /// Right now we only persist whether the backfill is finished and the corresponding cdc offset
74    /// schema: | `split_id` | `pk...` | `backfill_finished` | `row_count` | `cdc_offset` |
75    ///
76    /// For parallelized cdc backfill:
77    /// schema: | `split_id` | `backfill_finished` | `row_count` | `cdc_offset_low` | `cdc_offset_high` |
78    pub fn build_backfill_state_catalog(
79        &self,
80        state: &mut BuildFragmentGraphState,
81        is_parallelized_backfill: bool,
82    ) -> TableCatalog {
83        if is_parallelized_backfill {
84            let mut catalog_builder = TableCatalogBuilder::default();
85            // Use `split_id` as primary key in state table.
86            catalog_builder.add_column(&Field::with_name(DataType::Int64, "split_id"));
87            catalog_builder.add_order_column(0, OrderType::ascending());
88            catalog_builder.add_column(&Field::with_name(DataType::Boolean, "backfill_finished"));
89            // `row_count` column, the number of rows read from snapshot
90            catalog_builder.add_column(&Field::with_name(DataType::Int64, "row_count"));
91            catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset_low"));
92            catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset_high"));
93            catalog_builder
94                .build(vec![], 1)
95                .with_id(state.gen_table_id_wrapped())
96        } else {
97            let mut catalog_builder = TableCatalogBuilder::default();
98            let upstream_schema = &self.core.get_table_columns();
99
100            // Use `split_id` as primary key in state table.
101            // Currently we only support single split for cdc backfill.
102            catalog_builder.add_column(&Field::with_name(DataType::Varchar, "split_id"));
103            catalog_builder.add_order_column(0, OrderType::ascending());
104
105            // pk columns
106            for col_order in self.core.primary_key() {
107                let col = &upstream_schema[col_order.column_index];
108                catalog_builder.add_column(&Field::from(col));
109            }
110
111            catalog_builder.add_column(&Field::with_name(DataType::Boolean, "backfill_finished"));
112
113            // `row_count` column, the number of rows read from snapshot
114            catalog_builder.add_column(&Field::with_name(DataType::Int64, "row_count"));
115
116            // The offset is only for observability, not for recovery right now
117            catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset"));
118
119            // leave dist key empty, since the cdc backfill executor is singleton
120            catalog_builder
121                .build(vec![], 1)
122                .with_id(state.gen_table_id_wrapped())
123        }
124    }
125}
126
127impl_plan_tree_node_for_leaf! { Stream, StreamCdcTableScan }
128
129impl Distill for StreamCdcTableScan {
130    fn distill<'a>(&self) -> XmlNode<'a> {
131        let verbose = self.base.ctx().is_explain_verbose();
132        let mut vec = Vec::with_capacity(4);
133        vec.push(("table", Pretty::from(self.core.table_name.clone())));
134        vec.push(("columns", self.core.columns_pretty(verbose)));
135
136        if verbose {
137            let pk = IndicesDisplay {
138                indices: self.stream_key().unwrap_or_default(),
139                schema: self.base.schema(),
140            };
141            vec.push(("pk", pk.distill()));
142            let dist = Pretty::display(&DistributionDisplay {
143                distribution: self.distribution(),
144                input_schema: self.base.schema(),
145            });
146            vec.push(("dist", dist));
147        }
148
149        childless_record("StreamCdcTableScan", vec)
150    }
151}
152
153impl StreamNode for StreamCdcTableScan {
154    fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody {
155        unreachable!(
156            "stream scan cannot be converted into a prost body -- call `adhoc_to_stream_prost` instead."
157        )
158    }
159}
160
161impl StreamCdcTableScan {
162    /// plan: merge -> filter -> exchange(simple) -> `stream_scan`
163    pub fn adhoc_to_stream_prost(
164        &self,
165        state: &mut BuildFragmentGraphState,
166    ) -> SchedulerResult<PbStreamNode> {
167        use risingwave_pb::stream_plan::*;
168
169        let stream_key = self
170            .stream_key()
171            .unwrap_or_else(|| {
172                panic!(
173                    "should always have a stream key in the stream plan but not, sub plan: {}",
174                    PlanRef::from(self.clone()).explain_to_string()
175                )
176            })
177            .iter()
178            .map(|x| *x as u32)
179            .collect_vec();
180
181        // The schema of the shared cdc source upstream is different from snapshot.
182        let cdc_source_schema = ColumnCatalog::debezium_cdc_source_cols()
183            .into_iter()
184            .map(|c| Field::from(c.column_desc).to_prost())
185            .collect_vec();
186
187        let catalog = self
188            .build_backfill_state_catalog(state, self.core.options.is_parallelized_backfill())
189            .to_internal_table_prost();
190
191        // We need to pass the id of upstream source job here
192        let upstream_source_id = self.core.cdc_table_desc.source_id;
193
194        // filter upstream source chunk by the value of `_rw_table_name` column
195        let connect_properties = &self.core.cdc_table_desc.connect_properties;
196        let canonical_name = dispatch_cdc_source_type!(
197            connect_properties.cdc_source_type(),
198            T,
199            T::canonical_name(connect_properties),
200            unreachable!("CDC table without a CDC connector")
201        );
202        let filter_expr = Self::build_cdc_filter_expr(&canonical_name);
203
204        let filter_operator_id = self.core.ctx.next_plan_node_id();
205        // The filter node receive chunks in `(payload, _rw_offset, _rw_table_name)` schema
206        let filter_stream_node = StreamNode {
207            operator_id: filter_operator_id.to_stream_node_operator_id(),
208            input: vec![
209                // The merge node body will be filled by the `ActorBuilder` on the meta service.
210                PbStreamNode {
211                    node_body: Some(PbNodeBody::Merge(Default::default())),
212                    identity: "Upstream".into(),
213                    fields: cdc_source_schema.clone(),
214                    stream_key: vec![], // not used
215                    ..Default::default()
216                },
217            ],
218            stream_key: vec![], // not used
219            stream_kind: PbStreamKind::AppendOnly as _,
220            identity: "StreamCdcFilter".to_owned(),
221            fields: cdc_source_schema.clone(),
222            node_body: Some(PbNodeBody::CdcFilter(Box::new(CdcFilterNode {
223                search_condition: Some(filter_expr.to_expr_proto()),
224                upstream_source_id,
225            }))),
226        };
227
228        let exchange_operator_id = self.core.ctx.next_plan_node_id();
229        let strategy = if self.core.options.is_parallelized_backfill() {
230            DispatchStrategy {
231                r#type: DispatcherType::Broadcast as _,
232                dist_key_indices: vec![],
233                output_mapping: PbDispatchOutputMapping::identical(cdc_source_schema.len()).into(),
234            }
235        } else {
236            DispatchStrategy {
237                r#type: DispatcherType::Simple as _,
238                dist_key_indices: vec![], // simple exchange doesn't need dist key
239                output_mapping: PbDispatchOutputMapping::identical(cdc_source_schema.len()).into(),
240            }
241        };
242        // Add a simple exchange node between filter and stream scan
243        let exchange_stream_node = StreamNode {
244            operator_id: exchange_operator_id.to_stream_node_operator_id(),
245            input: vec![filter_stream_node],
246            stream_key: vec![], // not used
247            stream_kind: PbStreamKind::AppendOnly as _,
248            identity: "Exchange".to_owned(),
249            fields: cdc_source_schema,
250            node_body: Some(PbNodeBody::Exchange(Box::new(ExchangeNode {
251                strategy: Some(strategy),
252            }))),
253        };
254
255        // The required columns from the external table
256        let upstream_column_ids = self
257            .core
258            .output_and_pk_column_ids()
259            .iter()
260            .map(ColumnId::get_id)
261            .collect_vec();
262
263        let output_indices = self
264            .core
265            .output_column_ids()
266            .iter()
267            .map(|i| {
268                upstream_column_ids
269                    .iter()
270                    .position(|&x| x == i.get_id())
271                    .unwrap() as u32
272            })
273            .collect_vec();
274
275        tracing::debug!(
276            output_column_ids=?self.core.output_column_ids(),
277            ?upstream_column_ids,
278            ?output_indices,
279            "stream cdc table scan output indices"
280        );
281
282        let options = self.core.options.to_proto();
283        let stream_scan_body = PbNodeBody::StreamCdcScan(Box::new(StreamCdcScanNode {
284            table_id: upstream_source_id.as_cdc_table_id(),
285            upstream_column_ids,
286            output_indices,
287            // The table desc used by backfill executor
288            state_table: Some(catalog),
289            cdc_table_desc: Some(self.core.cdc_table_desc.to_protobuf()),
290            rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit,
291            disable_backfill: options.disable_backfill,
292            options: Some(options),
293        }));
294
295        // plan: merge -> filter -> exchange(simple) -> stream_scan
296        Ok(PbStreamNode {
297            fields: self.schema().to_prost(),
298            input: vec![exchange_stream_node],
299            node_body: Some(stream_scan_body),
300            stream_key,
301            operator_id: self.base.id().to_stream_node_operator_id(),
302            identity: self.distill_to_string(),
303            stream_kind: self.stream_kind().to_protobuf() as i32,
304        })
305    }
306
307    // The filter node receive input chunks in `(payload, _rw_offset, _rw_table_name)` schema
308    pub fn build_cdc_filter_expr(canonical_name: &str) -> ExprImpl {
309        FunctionCall::new(
310            ExprType::Equal,
311            vec![
312                InputRef::new(2, DataType::Varchar).into(),
313                ExprImpl::literal_varchar(canonical_name.into()),
314            ],
315        )
316        .unwrap()
317        .into()
318    }
319}
320
321impl ExprRewritable<Stream> for StreamCdcTableScan {
322    fn has_rewritable_expr(&self) -> bool {
323        true
324    }
325
326    fn rewrite_exprs(&self, r: &mut dyn ExprRewriter) -> PlanRef {
327        let core = self.core.clone();
328        core.rewrite_exprs(r);
329        Self::new(core).into()
330    }
331}
332
333impl ExprVisitable for StreamCdcTableScan {
334    fn visit_exprs(&self, v: &mut dyn ExprVisitor) {
335        self.core.visit_exprs(v);
336    }
337}
338
339#[cfg(test)]
340mod tests {
341    use std::str::FromStr;
342
343    use risingwave_common::row::OwnedRow;
344    use risingwave_common::types::{JsonbVal, ScalarImpl};
345
346    use super::*;
347
348    #[tokio::test]
349    async fn test_cdc_filter_expr() {
350        let t1_json = JsonbVal::from_str(r#"{ "before": null, "after": { "v": 111, "v2": 222.2 }, "source": { "version": "2.2.0.Alpha3", "connector": "mysql", "name": "dbserver1", "ts_ms": 1678428689000, "snapshot": "false", "db": "inventory", "sequence": null, "table": "t1", "server_id": 223344, "gtid": null, "file": "mysql-bin.000003", "pos": 774, "row": 0, "thread": 8, "query": null }, "op": "c", "ts_ms": 1678428689389, "transaction": null }"#).unwrap();
351        let t2_json = JsonbVal::from_str(r#"{ "before": null, "after": { "v": 333, "v2": 666.6 }, "source": { "version": "2.2.0.Alpha3", "connector": "mysql", "name": "dbserver1", "ts_ms": 1678428689000, "snapshot": "false", "db": "inventory", "sequence": null, "table": "t2", "server_id": 223344, "gtid": null, "file": "mysql-bin.000003", "pos": 884, "row": 0, "thread": 8, "query": null }, "op": "c", "ts_ms": 1678428689389, "transaction": null }"#).unwrap();
352
353        // NOTES: transaction metadata column expects to be filtered out before going to cdc filter
354        let trx_json = JsonbVal::from_str(r#"{"data_collections": null, "event_count": null, "id": "35319:3962662584", "status": "BEGIN", "ts_ms": 1704263537068}"#).unwrap();
355        let row1 = OwnedRow::new(vec![
356            Some(t1_json.into()),
357            Some(r#"{"file": "1.binlog", "pos": 100}"#.into()),
358            Some("public.t2".into()),
359        ]);
360        let row2 = OwnedRow::new(vec![
361            Some(t2_json.into()),
362            Some(r#"{"file": "2.binlog", "pos": 100}"#.into()),
363            Some("abs.t2".into()),
364        ]);
365
366        let row3 = OwnedRow::new(vec![
367            Some(trx_json.into()),
368            Some(r#"{"file": "3.binlog", "pos": 100}"#.into()),
369            Some("public.t2".into()),
370        ]);
371
372        let filter_expr = StreamCdcTableScan::build_cdc_filter_expr("public.t2");
373        assert_eq!(
374            filter_expr.eval_row(&row1).await.unwrap(),
375            Some(ScalarImpl::Bool(true))
376        );
377        assert_eq!(
378            filter_expr.eval_row(&row2).await.unwrap(),
379            Some(ScalarImpl::Bool(false))
380        );
381        assert_eq!(
382            filter_expr.eval_row(&row3).await.unwrap(),
383            Some(ScalarImpl::Bool(true))
384        )
385    }
386}