1use itertools::Itertools;
16use pretty_xmlish::{Pretty, XmlNode};
17use risingwave_common::catalog::{ColumnCatalog, Field};
18use risingwave_common::types::DataType;
19use risingwave_common::util::sort_util::OrderType;
20use risingwave_connector::source::cdc::CdcSourceExternalTableIdentifier;
21use risingwave_connector::{WithPropertiesExt, dispatch_cdc_source_type};
22use risingwave_pb::stream_plan::PbStreamNode;
23use risingwave_pb::stream_plan::stream_node::{PbNodeBody, PbStreamKind};
24
25use super::stream::prelude::*;
26use super::utils::{Distill, childless_record};
27use super::{ExprRewritable, PlanBase, StreamNode, StreamPlanRef as PlanRef, generic};
28use crate::catalog::ColumnId;
29use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprType, ExprVisitor, FunctionCall, InputRef};
30use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
31use crate::optimizer::plan_node::utils::{IndicesDisplay, TableCatalogBuilder};
32use crate::optimizer::property::{Distribution, DistributionDisplay};
33use crate::scheduler::SchedulerResult;
34use crate::stream_fragmenter::BuildFragmentGraphState;
35use crate::{Explain, TableCatalog};
36
37#[derive(Debug, Clone, PartialEq, Eq, Hash)]
40pub struct StreamCdcTableScan {
41 pub base: PlanBase<Stream>,
42 core: generic::CdcScan,
43}
44
45impl StreamCdcTableScan {
46 pub fn new(core: generic::CdcScan) -> Self {
47 let distribution = Distribution::SomeShard;
48 let base = PlanBase::new_stream_with_core(
49 &core,
50 distribution,
51 StreamKind::Upsert,
55 false,
56 core.watermark_columns(),
57 core.columns_monotonicity(),
58 );
59 Self { base, core }
60 }
61
62 pub fn table_name(&self) -> &str {
63 &self.core.table_name
64 }
65
66 pub fn core(&self) -> &generic::CdcScan {
67 &self.core
68 }
69
70 pub fn build_backfill_state_catalog(
79 &self,
80 state: &mut BuildFragmentGraphState,
81 is_parallelized_backfill: bool,
82 ) -> TableCatalog {
83 if is_parallelized_backfill {
84 let mut catalog_builder = TableCatalogBuilder::default();
85 catalog_builder.add_column(&Field::with_name(DataType::Int64, "split_id"));
87 catalog_builder.add_order_column(0, OrderType::ascending());
88 catalog_builder.add_column(&Field::with_name(DataType::Boolean, "backfill_finished"));
89 catalog_builder.add_column(&Field::with_name(DataType::Int64, "row_count"));
91 catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset_low"));
92 catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset_high"));
93 catalog_builder
94 .build(vec![], 1)
95 .with_id(state.gen_table_id_wrapped())
96 } else {
97 let mut catalog_builder = TableCatalogBuilder::default();
98 let upstream_schema = &self.core.get_table_columns();
99
100 catalog_builder.add_column(&Field::with_name(DataType::Varchar, "split_id"));
103 catalog_builder.add_order_column(0, OrderType::ascending());
104
105 for col_order in self.core.primary_key() {
107 let col = &upstream_schema[col_order.column_index];
108 catalog_builder.add_column(&Field::from(col));
109 }
110
111 catalog_builder.add_column(&Field::with_name(DataType::Boolean, "backfill_finished"));
112
113 catalog_builder.add_column(&Field::with_name(DataType::Int64, "row_count"));
115
116 catalog_builder.add_column(&Field::with_name(DataType::Jsonb, "cdc_offset"));
118
119 catalog_builder
121 .build(vec![], 1)
122 .with_id(state.gen_table_id_wrapped())
123 }
124 }
125}
126
127impl_plan_tree_node_for_leaf! { Stream, StreamCdcTableScan }
128
129impl Distill for StreamCdcTableScan {
130 fn distill<'a>(&self) -> XmlNode<'a> {
131 let verbose = self.base.ctx().is_explain_verbose();
132 let mut vec = Vec::with_capacity(4);
133 vec.push(("table", Pretty::from(self.core.table_name.clone())));
134 vec.push(("columns", self.core.columns_pretty(verbose)));
135
136 if verbose {
137 let pk = IndicesDisplay {
138 indices: self.stream_key().unwrap_or_default(),
139 schema: self.base.schema(),
140 };
141 vec.push(("pk", pk.distill()));
142 let dist = Pretty::display(&DistributionDisplay {
143 distribution: self.distribution(),
144 input_schema: self.base.schema(),
145 });
146 vec.push(("dist", dist));
147 }
148
149 childless_record("StreamCdcTableScan", vec)
150 }
151}
152
153impl StreamNode for StreamCdcTableScan {
154 fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody {
155 unreachable!(
156 "stream scan cannot be converted into a prost body -- call `adhoc_to_stream_prost` instead."
157 )
158 }
159}
160
161impl StreamCdcTableScan {
162 pub fn adhoc_to_stream_prost(
164 &self,
165 state: &mut BuildFragmentGraphState,
166 ) -> SchedulerResult<PbStreamNode> {
167 use risingwave_pb::stream_plan::*;
168
169 let stream_key = self
170 .stream_key()
171 .unwrap_or_else(|| {
172 panic!(
173 "should always have a stream key in the stream plan but not, sub plan: {}",
174 PlanRef::from(self.clone()).explain_to_string()
175 )
176 })
177 .iter()
178 .map(|x| *x as u32)
179 .collect_vec();
180
181 let cdc_source_schema = ColumnCatalog::debezium_cdc_source_cols()
183 .into_iter()
184 .map(|c| Field::from(c.column_desc).to_prost())
185 .collect_vec();
186
187 let catalog = self
188 .build_backfill_state_catalog(state, self.core.options.is_parallelized_backfill())
189 .to_internal_table_prost();
190
191 let upstream_source_id = self.core.cdc_table_desc.source_id;
193
194 let connect_properties = &self.core.cdc_table_desc.connect_properties;
196 let canonical_name = dispatch_cdc_source_type!(
197 connect_properties.cdc_source_type(),
198 T,
199 T::canonical_name(connect_properties),
200 unreachable!("CDC table without a CDC connector")
201 );
202 let filter_expr = Self::build_cdc_filter_expr(&canonical_name);
203
204 let filter_operator_id = self.core.ctx.next_plan_node_id();
205 let filter_stream_node = StreamNode {
207 operator_id: filter_operator_id.to_stream_node_operator_id(),
208 input: vec![
209 PbStreamNode {
211 node_body: Some(PbNodeBody::Merge(Default::default())),
212 identity: "Upstream".into(),
213 fields: cdc_source_schema.clone(),
214 stream_key: vec![], ..Default::default()
216 },
217 ],
218 stream_key: vec![], stream_kind: PbStreamKind::AppendOnly as _,
220 identity: "StreamCdcFilter".to_owned(),
221 fields: cdc_source_schema.clone(),
222 node_body: Some(PbNodeBody::CdcFilter(Box::new(CdcFilterNode {
223 search_condition: Some(filter_expr.to_expr_proto()),
224 upstream_source_id,
225 }))),
226 };
227
228 let exchange_operator_id = self.core.ctx.next_plan_node_id();
229 let strategy = if self.core.options.is_parallelized_backfill() {
230 DispatchStrategy {
231 r#type: DispatcherType::Broadcast as _,
232 dist_key_indices: vec![],
233 output_mapping: PbDispatchOutputMapping::identical(cdc_source_schema.len()).into(),
234 }
235 } else {
236 DispatchStrategy {
237 r#type: DispatcherType::Simple as _,
238 dist_key_indices: vec![], output_mapping: PbDispatchOutputMapping::identical(cdc_source_schema.len()).into(),
240 }
241 };
242 let exchange_stream_node = StreamNode {
244 operator_id: exchange_operator_id.to_stream_node_operator_id(),
245 input: vec![filter_stream_node],
246 stream_key: vec![], stream_kind: PbStreamKind::AppendOnly as _,
248 identity: "Exchange".to_owned(),
249 fields: cdc_source_schema,
250 node_body: Some(PbNodeBody::Exchange(Box::new(ExchangeNode {
251 strategy: Some(strategy),
252 }))),
253 };
254
255 let upstream_column_ids = self
257 .core
258 .output_and_pk_column_ids()
259 .iter()
260 .map(ColumnId::get_id)
261 .collect_vec();
262
263 let output_indices = self
264 .core
265 .output_column_ids()
266 .iter()
267 .map(|i| {
268 upstream_column_ids
269 .iter()
270 .position(|&x| x == i.get_id())
271 .unwrap() as u32
272 })
273 .collect_vec();
274
275 tracing::debug!(
276 output_column_ids=?self.core.output_column_ids(),
277 ?upstream_column_ids,
278 ?output_indices,
279 "stream cdc table scan output indices"
280 );
281
282 let options = self.core.options.to_proto();
283 let stream_scan_body = PbNodeBody::StreamCdcScan(Box::new(StreamCdcScanNode {
284 table_id: upstream_source_id.as_cdc_table_id(),
285 upstream_column_ids,
286 output_indices,
287 state_table: Some(catalog),
289 cdc_table_desc: Some(self.core.cdc_table_desc.to_protobuf()),
290 rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit,
291 disable_backfill: options.disable_backfill,
292 options: Some(options),
293 }));
294
295 Ok(PbStreamNode {
297 fields: self.schema().to_prost(),
298 input: vec![exchange_stream_node],
299 node_body: Some(stream_scan_body),
300 stream_key,
301 operator_id: self.base.id().to_stream_node_operator_id(),
302 identity: self.distill_to_string(),
303 stream_kind: self.stream_kind().to_protobuf() as i32,
304 })
305 }
306
307 pub fn build_cdc_filter_expr(canonical_name: &str) -> ExprImpl {
309 FunctionCall::new(
310 ExprType::Equal,
311 vec![
312 InputRef::new(2, DataType::Varchar).into(),
313 ExprImpl::literal_varchar(canonical_name.into()),
314 ],
315 )
316 .unwrap()
317 .into()
318 }
319}
320
321impl ExprRewritable<Stream> for StreamCdcTableScan {
322 fn has_rewritable_expr(&self) -> bool {
323 true
324 }
325
326 fn rewrite_exprs(&self, r: &mut dyn ExprRewriter) -> PlanRef {
327 let core = self.core.clone();
328 core.rewrite_exprs(r);
329 Self::new(core).into()
330 }
331}
332
333impl ExprVisitable for StreamCdcTableScan {
334 fn visit_exprs(&self, v: &mut dyn ExprVisitor) {
335 self.core.visit_exprs(v);
336 }
337}
338
339#[cfg(test)]
340mod tests {
341 use std::str::FromStr;
342
343 use risingwave_common::row::OwnedRow;
344 use risingwave_common::types::{JsonbVal, ScalarImpl};
345
346 use super::*;
347
348 #[tokio::test]
349 async fn test_cdc_filter_expr() {
350 let t1_json = JsonbVal::from_str(r#"{ "before": null, "after": { "v": 111, "v2": 222.2 }, "source": { "version": "2.2.0.Alpha3", "connector": "mysql", "name": "dbserver1", "ts_ms": 1678428689000, "snapshot": "false", "db": "inventory", "sequence": null, "table": "t1", "server_id": 223344, "gtid": null, "file": "mysql-bin.000003", "pos": 774, "row": 0, "thread": 8, "query": null }, "op": "c", "ts_ms": 1678428689389, "transaction": null }"#).unwrap();
351 let t2_json = JsonbVal::from_str(r#"{ "before": null, "after": { "v": 333, "v2": 666.6 }, "source": { "version": "2.2.0.Alpha3", "connector": "mysql", "name": "dbserver1", "ts_ms": 1678428689000, "snapshot": "false", "db": "inventory", "sequence": null, "table": "t2", "server_id": 223344, "gtid": null, "file": "mysql-bin.000003", "pos": 884, "row": 0, "thread": 8, "query": null }, "op": "c", "ts_ms": 1678428689389, "transaction": null }"#).unwrap();
352
353 let trx_json = JsonbVal::from_str(r#"{"data_collections": null, "event_count": null, "id": "35319:3962662584", "status": "BEGIN", "ts_ms": 1704263537068}"#).unwrap();
355 let row1 = OwnedRow::new(vec![
356 Some(t1_json.into()),
357 Some(r#"{"file": "1.binlog", "pos": 100}"#.into()),
358 Some("public.t2".into()),
359 ]);
360 let row2 = OwnedRow::new(vec![
361 Some(t2_json.into()),
362 Some(r#"{"file": "2.binlog", "pos": 100}"#.into()),
363 Some("abs.t2".into()),
364 ]);
365
366 let row3 = OwnedRow::new(vec![
367 Some(trx_json.into()),
368 Some(r#"{"file": "3.binlog", "pos": 100}"#.into()),
369 Some("public.t2".into()),
370 ]);
371
372 let filter_expr = StreamCdcTableScan::build_cdc_filter_expr("public.t2");
373 assert_eq!(
374 filter_expr.eval_row(&row1).await.unwrap(),
375 Some(ScalarImpl::Bool(true))
376 );
377 assert_eq!(
378 filter_expr.eval_row(&row2).await.unwrap(),
379 Some(ScalarImpl::Bool(false))
380 );
381 assert_eq!(
382 filter_expr.eval_row(&row3).await.unwrap(),
383 Some(ScalarImpl::Bool(true))
384 )
385 }
386}