1use std::collections::BTreeMap;
16use std::pin::Pin;
17
18use either::Either;
19use futures::stream;
20use futures::stream::select_with_strategy;
21use itertools::Itertools;
22use risingwave_common::array::{DataChunk, Op};
23use risingwave_common::bail;
24use risingwave_common::bitmap::BitmapBuilder;
25use risingwave_common::catalog::{CdcKeyComparison, ColumnDesc};
26use risingwave_common::row::RowExt;
27use risingwave_common::util::sort_util::OrderType;
28use risingwave_connector::WithPropertiesExt;
29use risingwave_connector::parser::{
30 BigintUnsignedHandlingMode, ByteStreamSourceParser, DebeziumParser, DebeziumProps,
31 EncodingProperties, JsonProperties, ProtocolProperties, SourceStreamChunkBuilder,
32 SpecificParserConfig, TimeHandling, TimestampHandling, TimestamptzHandling,
33};
34use risingwave_connector::source::cdc::CdcScanOptions;
35use risingwave_connector::source::cdc::external::{
36 CdcOffset, CdcOffsetParseFunc, ExternalCdcTableType, ExternalTableReaderImpl,
37};
38use risingwave_connector::source::{SourceColumnDesc, SourceContext, SourceCtrlOpts};
39use rw_futures_util::pausable;
40use thiserror_ext::AsReport;
41
42use crate::executor::backfill::cdc::state::CdcBackfillState;
43use crate::executor::backfill::cdc::upstream_table::external::ExternalStorageTable;
44use crate::executor::backfill::cdc::upstream_table::reader::{
45 build_reader_and_poll_upstream, create_table_reader_with_retry,
46};
47use crate::executor::backfill::cdc::upstream_table::snapshot::{
48 SnapshotReadArgs, UpstreamTableRead, UpstreamTableReader,
49};
50use crate::executor::backfill::utils::{
51 cmp_pk_unsigned_aware, get_cdc_chunk_last_offset, get_new_pos, mapping_chunk, mapping_message,
52 mark_cdc_chunk,
53};
54use crate::executor::monitor::CdcBackfillMetrics;
55use crate::executor::prelude::*;
56use crate::executor::{Mutation, TableId};
57use crate::task::CreateMviewProgressReporter;
58
59const METADATA_STATE_LEN: usize = 4;
61
62struct PkCompareInfo<'a> {
63 indices: &'a [usize],
64 order: &'a [OrderType],
65 needs_unsigned_i64_compare: &'a [bool],
66}
67
68fn can_poll_upstream_while_creating_reader(
69 current_pk_pos: Option<&OwnedRow>,
70 pk_comparisons_are_known: bool,
71) -> bool {
72 current_pk_pos.is_none() || pk_comparisons_are_known
73}
74
75enum NoSnapshotUpstreamOutcome {
77 Continue(Option<Message>),
80 Stop(Barrier),
82}
83
84#[derive(Clone, Copy)]
87struct SnapshotCursor<'a> {
88 pk_pos: &'a OwnedRow,
89 pk_needs_unsigned_i64_compare: &'a [bool],
90}
91
92struct NoSnapshotUpstreamHandler<'a, S: StateStore> {
93 actor_ctx: &'a ActorContextRef,
94 metrics: &'a CdcBackfillMetrics,
95 table_id: TableId,
96 upstream_table_name: &'a str,
97 offset_parse_func: &'a mut CdcOffsetParseFunc,
98 output_indices: &'a [usize],
99 pk_indices: &'a [usize],
100 pk_order: &'a [OrderType],
101 cursor: Option<SnapshotCursor<'a>>,
103 total_snapshot_row_count: u64,
104 state_impl: &'a mut CdcBackfillState<S>,
105 last_binlog_offset: &'a mut Option<CdcOffset>,
106 is_snapshot_paused: &'a mut bool,
107 rate_limit_rps: &'a mut Option<u32>,
108}
109
110impl<S: StateStore> NoSnapshotUpstreamHandler<'_, S> {
111 async fn handle_upstream(
112 &mut self,
113 msg: Message,
114 ) -> StreamExecutorResult<NoSnapshotUpstreamOutcome> {
115 match msg {
116 Message::Barrier(barrier) => {
117 if let Some(mutation) = barrier.mutation.as_deref() {
118 match mutation {
119 Mutation::Pause => {
120 *self.is_snapshot_paused = true;
121 tracing::info!(
122 table_id = %self.table_id,
123 upstream_table_name = self.upstream_table_name,
124 "snapshot is paused by barrier"
125 );
126 }
127 Mutation::Resume => {
128 *self.is_snapshot_paused = false;
129 tracing::info!(
130 table_id = %self.table_id,
131 upstream_table_name = self.upstream_table_name,
132 "snapshot is resumed by barrier"
133 );
134 }
135 Mutation::Throttle(_) => {
136 if let Some(entry) =
137 mutation.backfill_throttle_config(self.actor_ctx.fragment_id)
138 {
139 *self.rate_limit_rps = entry.rate_limit;
140 }
141 }
142 mutation if mutation.is_stop(self.actor_ctx.id) => {
143 tracing::info!(
144 table_id = %self.table_id,
145 upstream_table_name = self.upstream_table_name,
146 "CdcBackfill has been dropped due to config change"
147 );
148 self.state_impl.commit_state(barrier.epoch).await?;
149
150 return Ok(NoSnapshotUpstreamOutcome::Stop(barrier));
151 }
152 _ => (),
153 }
154 }
155
156 self.state_impl
158 .mutate_and_commit_state(
159 self.cursor.map(|cursor| cursor.pk_pos.clone()),
160 self.last_binlog_offset.clone(),
161 self.total_snapshot_row_count,
162 false,
163 barrier.epoch,
164 )
165 .await?;
166
167 Ok(NoSnapshotUpstreamOutcome::Continue(Some(Message::Barrier(
168 barrier,
169 ))))
170 }
171 Message::Chunk(chunk) => {
172 let Some(cursor) = self.cursor else {
173 return Ok(NoSnapshotUpstreamOutcome::Continue(None));
175 };
176
177 if !chunk.has_visible_rows() {
178 return Ok(NoSnapshotUpstreamOutcome::Continue(None));
179 }
180
181 let (chunk, consumed_offset) = CdcBackfillExecutor::<S>::filter_recovery_chunk(
182 self.offset_parse_func,
183 chunk,
184 cursor.pk_pos,
185 PkCompareInfo {
186 indices: self.pk_indices,
187 order: self.pk_order,
188 needs_unsigned_i64_compare: cursor.pk_needs_unsigned_i64_compare,
189 },
190 self.last_binlog_offset,
191 self.output_indices,
192 )?;
193
194 if let Some(consumed_offset) = consumed_offset {
195 *self.last_binlog_offset = Some(consumed_offset);
196 }
197
198 let chunk = chunk.map(|chunk| {
199 CdcBackfillExecutor::<S>::report_metrics(
200 self.metrics,
201 0,
202 chunk.cardinality() as u64,
203 );
204
205 Message::Chunk(chunk)
206 });
207
208 Ok(NoSnapshotUpstreamOutcome::Continue(chunk))
209 }
210 Message::Watermark(_) => Ok(NoSnapshotUpstreamOutcome::Continue(None)),
211 }
212 }
213
214 async fn wait_for_reader(
215 &mut self,
216 upstream: &mut (impl Stream<Item = StreamExecutorResult<Message>> + Unpin),
217 future: &mut Pin<Box<impl Future<Output = ExternalTableReaderImpl>>>,
218 ) -> StreamExecutorResult<Either<NoSnapshotUpstreamOutcome, ExternalTableReaderImpl>> {
219 match build_reader_and_poll_upstream(upstream, future).await? {
220 Either::Left(msg) => Ok(Either::Left(self.handle_upstream(msg).await?)),
221 Either::Right(table_reader) => Ok(Either::Right(table_reader)),
222 }
223 }
224}
225
226pub(crate) fn get_cdc_json_parse_handling_from_properties(
231 properties: &BTreeMap<String, String>,
232) -> (
233 Option<TimestampHandling>,
234 Option<TimestamptzHandling>,
235 Option<TimeHandling>,
236 Option<BigintUnsignedHandlingMode>,
237) {
238 let (timestamp_handling, timestamptz_handling, time_handling) = match properties
239 .get("debezium.time.precision.mode")
240 {
241 None => (
242 Some(TimestampHandling::Micro),
243 Some(TimestamptzHandling::Micro),
244 Some(TimeHandling::Micro),
245 ),
246 Some(m) if m == "connect" => (
247 Some(TimestampHandling::Milli),
248 Some(TimestamptzHandling::Milli),
249 Some(TimeHandling::Milli),
250 ),
251 Some(other) => {
252 tracing::warn!(
254 "Unsupported debezium.time.precision.mode = {other}, fall back to default parser."
255 );
256 (None, None, None)
257 }
258 };
259 let bigint_unsigned_handling = properties
260 .get("debezium.bigint.unsigned.handling.mode")
261 .is_some_and(|v| v == "precise")
262 .then_some(BigintUnsignedHandlingMode::Precise);
263 (
264 timestamp_handling,
265 timestamptz_handling,
266 time_handling,
267 bigint_unsigned_handling,
268 )
269}
270
271pub struct CdcBackfillExecutor<S: StateStore> {
272 actor_ctx: ActorContextRef,
273
274 external_table: ExternalStorageTable,
276
277 upstream: Executor,
279
280 output_indices: Vec<usize>,
282
283 output_columns: Vec<ColumnDesc>,
285
286 state_impl: CdcBackfillState<S>,
288
289 progress: Option<CreateMviewProgressReporter>,
292
293 metrics: CdcBackfillMetrics,
294
295 rate_limit_rps: Option<u32>,
297
298 options: CdcScanOptions,
299
300 properties: BTreeMap<String, String>,
301}
302
303impl<S: StateStore> CdcBackfillExecutor<S> {
304 #[expect(clippy::too_many_arguments)]
305 pub fn new(
306 actor_ctx: ActorContextRef,
307 external_table: ExternalStorageTable,
308 upstream: Executor,
309 output_indices: Vec<usize>,
310 output_columns: Vec<ColumnDesc>,
311 progress: Option<CreateMviewProgressReporter>,
312 metrics: Arc<StreamingMetrics>,
313 state_table: StateTable<S>,
314 rate_limit_rps: Option<u32>,
315 options: CdcScanOptions,
316 properties: BTreeMap<String, String>,
317 ) -> Self {
318 let pk_indices = external_table.pk_indices();
319 let upstream_table_id = external_table.table_id();
320 let state_impl = CdcBackfillState::new(
321 upstream_table_id,
322 state_table,
323 pk_indices.len() + METADATA_STATE_LEN,
324 );
325
326 let metrics = metrics.new_cdc_backfill_metrics(external_table.table_id(), actor_ctx.id);
327 Self {
328 actor_ctx,
329 external_table,
330 upstream,
331 output_indices,
332 output_columns,
333 state_impl,
334 progress,
335 metrics,
336 rate_limit_rps,
337 options,
338 properties,
339 }
340 }
341
342 fn report_metrics(
343 metrics: &CdcBackfillMetrics,
344 snapshot_processed_row_count: u64,
345 upstream_processed_row_count: u64,
346 ) {
347 metrics
348 .cdc_backfill_snapshot_read_row_count
349 .inc_by(snapshot_processed_row_count);
350
351 metrics
352 .cdc_backfill_upstream_output_row_count
353 .inc_by(upstream_processed_row_count);
354 }
355
356 fn consume_upstream_chunk_buffer(
362 offset_parse_func: &CdcOffsetParseFunc,
363 upstream_chunk_buffer: &mut Vec<StreamChunk>,
364 current_pk_pos: Option<&OwnedRow>,
365 pk_compare: PkCompareInfo<'_>,
366 last_binlog_offset: &Option<CdcOffset>,
367 output_indices: &[usize],
368 ) -> StreamExecutorResult<(Vec<StreamChunk>, u64, Option<CdcOffset>)> {
369 let Some(current_pos) = current_pk_pos else {
370 return Ok((vec![], 0, None));
371 };
372
373 let buffered_chunks = std::mem::take(upstream_chunk_buffer);
374 let mut emitted_chunks = Vec::with_capacity(buffered_chunks.len());
375 let mut upstream_processed_row_count = 0;
376 let mut consumed_binlog_offset = None;
377 let mut retained_chunks = Vec::with_capacity(buffered_chunks.len());
378 let mut can_advance_consumed_offset = true;
379
380 for chunk in buffered_chunks {
381 let mut emitted_vis = BitmapBuilder::zeroed(chunk.capacity());
382 let mut retained_vis = BitmapBuilder::zeroed(chunk.capacity());
383 for idx in 0..chunk.capacity() {
384 let (op, row, visible) = chunk.row_at(idx);
385 if !visible {
386 continue;
388 }
389 let event_offset = (*offset_parse_func)(
390 row.iter()
391 .last()
392 .flatten()
393 .expect("cdc offset must exist")
394 .into_utf8(),
395 )?;
396 let in_binlog_range = last_binlog_offset
397 .as_ref()
398 .is_none_or(|binlog_low| *binlog_low <= event_offset);
399
400 let row_pk = row.project(pk_compare.indices);
401 let reached_current_pos = cmp_pk_unsigned_aware(
402 row_pk.iter(),
403 current_pos.iter(),
404 pk_compare.order,
405 pk_compare.needs_unsigned_i64_compare,
406 )
407 .is_le();
408 if !in_binlog_range {
409 continue;
410 }
411 let should_emit = reached_current_pos;
412
413 match op {
414 Op::Insert | Op::Delete => {
415 if should_emit {
416 emitted_vis.set(idx, true);
417 if can_advance_consumed_offset {
418 consumed_binlog_offset = Some(event_offset);
419 }
420 upstream_processed_row_count += 1;
421 } else {
422 retained_vis.set(idx, true);
423 can_advance_consumed_offset = false;
424 }
425 }
426 Op::UpdateDelete | Op::UpdateInsert => {
427 unreachable!("CDC buffered chunks should not contain update pairs")
428 }
429 }
430 }
431
432 let emitted_vis = emitted_vis.finish();
433 if emitted_vis.count_ones() > 0 {
434 emitted_chunks.push(mapping_chunk(
435 chunk.clone_with_vis(emitted_vis),
436 output_indices,
437 ));
438 }
439
440 let retained_vis = retained_vis.finish();
441 if retained_vis.count_ones() > 0 {
442 retained_chunks.push(chunk.clone_with_vis(retained_vis));
443 }
444 }
445
446 *upstream_chunk_buffer = retained_chunks;
447
448 Ok((
449 emitted_chunks,
450 upstream_processed_row_count,
451 consumed_binlog_offset,
452 ))
453 }
454
455 fn filter_recovery_chunk(
459 offset_parse_func: &CdcOffsetParseFunc,
460 chunk: StreamChunk,
461 current_pos: &OwnedRow,
462 pk_compare: PkCompareInfo<'_>,
463 last_binlog_offset: &Option<CdcOffset>,
464 output_indices: &[usize],
465 ) -> StreamExecutorResult<(Option<StreamChunk>, Option<CdcOffset>)> {
466 let chunk_offset = get_cdc_chunk_last_offset(offset_parse_func, &chunk)?;
467 let chunk = mapping_chunk(
468 mark_cdc_chunk(
469 offset_parse_func,
470 chunk,
471 current_pos,
472 pk_compare.indices,
473 pk_compare.order,
474 pk_compare.needs_unsigned_i64_compare,
475 last_binlog_offset.clone(),
476 )?,
477 output_indices,
478 );
479 let chunk = (chunk.cardinality() > 0).then_some(chunk);
480 let consumed_offset = chunk_offset.filter(|chunk_offset| {
481 last_binlog_offset
482 .as_ref()
483 .is_none_or(|last| *last < *chunk_offset)
484 });
485 Ok((chunk, consumed_offset))
486 }
487
488 #[try_stream(ok = Message, error = StreamExecutorError)]
489 async fn execute_inner(mut self) {
490 let pk_indices = self.external_table.pk_indices().to_vec();
492 let pk_order = self.external_table.pk_order_types().to_vec();
493 let pk_needs_unsigned_i64_compare =
494 self.external_table.pk_comparisons().map(|comparisons| {
495 comparisons
496 .iter()
497 .map(|comparison| *comparison == CdcKeyComparison::UnsignedInt64)
498 .collect_vec()
499 });
500
501 let table_id = self.external_table.table_id();
502 let upstream_table_name = self.external_table.qualified_table_name();
503 let schema_table_name = self.external_table.schema_table_name().clone();
504 let external_database_name = self.external_table.database_name().to_owned();
505
506 let additional_columns = self
507 .output_columns
508 .iter()
509 .filter(|col| col.additional_column.column_type.is_some())
510 .cloned()
511 .collect_vec();
512
513 let mut upstream = self.upstream.execute();
514
515 let mut current_pk_pos: Option<OwnedRow>;
518
519 let first_barrier = expect_first_barrier(&mut upstream).await?;
521
522 let mut is_snapshot_paused = first_barrier.is_pause_on_startup();
523 let first_barrier_epoch = first_barrier.epoch;
524 yield Message::Barrier(first_barrier);
526
527 let mut state_impl = self.state_impl;
528
529 state_impl.init_epoch(first_barrier_epoch).await?;
530
531 let state = state_impl.restore_state().await?;
533 current_pk_pos = state.current_pk_pos.clone();
534
535 let need_backfill = !self.options.disable_backfill && !state.is_finished;
536
537 let mut total_snapshot_row_count = state.row_count as u64;
539
540 let handle_postgres_toast_columns: bool =
542 self.external_table.table_type() == &ExternalCdcTableType::Postgres;
543 let mut upstream = transform_upstream(
545 upstream,
546 self.output_columns.clone(),
547 self.properties.clone(),
548 handle_postgres_toast_columns,
549 )
550 .boxed()
551 .peekable();
552 let mut last_binlog_offset = state.last_cdc_offset.clone();
553
554 if need_backfill {
575 let mut offset_parse_func = self.external_table.table_type().get_cdc_offset_parser()?;
583 let mut future = Box::pin(create_table_reader_with_retry(
584 self.external_table.clone(),
585 self.actor_ctx.id,
586 self.actor_ctx.fragment_id,
587 ));
588 let table_reader = if can_poll_upstream_while_creating_reader(
589 current_pk_pos.as_ref(),
590 pk_needs_unsigned_i64_compare.is_some(),
591 ) {
592 let mut handler = NoSnapshotUpstreamHandler {
593 actor_ctx: &self.actor_ctx,
594 metrics: &self.metrics,
595 table_id,
596 upstream_table_name: &upstream_table_name,
597 offset_parse_func: &mut offset_parse_func,
598 output_indices: &self.output_indices,
599 pk_indices: &pk_indices,
600 pk_order: &pk_order,
601 cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
602 pk_pos,
603 pk_needs_unsigned_i64_compare: pk_needs_unsigned_i64_compare
604 .as_deref()
605 .expect("checked by can_poll_upstream_while_creating_reader"),
606 }),
607 total_snapshot_row_count,
608 state_impl: &mut state_impl,
609 last_binlog_offset: &mut last_binlog_offset,
610 is_snapshot_paused: &mut is_snapshot_paused,
611 rate_limit_rps: &mut self.rate_limit_rps,
612 };
613 loop {
614 match handler.wait_for_reader(&mut upstream, &mut future).await? {
615 Either::Left(NoSnapshotUpstreamOutcome::Continue(msg)) => {
616 if let Some(msg) = msg {
617 yield msg;
618 }
619 }
620 Either::Left(NoSnapshotUpstreamOutcome::Stop(barrier)) => {
621 yield Message::Barrier(barrier);
622
623 return Ok(());
624 }
625 Either::Right(table_reader) => break table_reader,
626 }
627 }
628 } else {
629 tracing::warn!(
630 %table_id,
631 upstream_table_name,
632 "waiting for the CDC table reader to recover legacy MySQL primary-key comparison metadata; checkpoint progress is paused"
633 );
634 future.as_mut().await
635 };
636 let pk_names = self.external_table.pk_names();
637 let pk_needs_unsigned_i64_compare = match pk_needs_unsigned_i64_compare {
638 Some(comparisons) => comparisons,
639 None => {
640 let comparisons = table_reader.pk_column_comparisons(&pk_names)?;
641 assert_eq!(comparisons.len(), pk_indices.len());
642 comparisons
643 .into_iter()
644 .map(|comparison| comparison == CdcKeyComparison::UnsignedInt64)
645 .collect_vec()
646 }
647 };
648 tracing::info!(
649 %table_id,
650 upstream_table_name,
651 "table reader created successfully"
652 );
653
654 let mut upstream_table_reader =
655 UpstreamTableReader::new(self.external_table.clone(), table_reader);
656
657 if last_binlog_offset.is_none() {
658 static CDC_CONN_SEMAPHORE: tokio::sync::Semaphore =
660 tokio::sync::Semaphore::const_new(10);
661
662 let _permit = CDC_CONN_SEMAPHORE.acquire().await.unwrap();
663 last_binlog_offset = upstream_table_reader.current_cdc_offset().await?;
664 }
665
666 tracing::info!(
667 %table_id,
668 upstream_table_name,
669 initial_binlog_offset = ?last_binlog_offset,
670 ?current_pk_pos,
671 is_finished = state.is_finished,
672 is_snapshot_paused,
673 snapshot_row_count = total_snapshot_row_count,
674 rate_limit = self.rate_limit_rps,
675 disable_backfill = self.options.disable_backfill,
676 snapshot_barrier_interval = self.options.snapshot_barrier_interval,
677 snapshot_batch_size = self.options.snapshot_batch_size,
678 "start cdc backfill",
679 );
680
681 let _ = Pin::new(&mut upstream).peek().await;
684
685 let mut handler = NoSnapshotUpstreamHandler {
687 actor_ctx: &self.actor_ctx,
688 metrics: &self.metrics,
689 table_id,
690 upstream_table_name: &upstream_table_name,
691 offset_parse_func: &mut offset_parse_func,
692 output_indices: &self.output_indices,
693 pk_indices: &pk_indices,
694 pk_order: &pk_order,
695 cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
696 pk_pos,
697 pk_needs_unsigned_i64_compare: &pk_needs_unsigned_i64_compare,
698 }),
699 total_snapshot_row_count,
700 state_impl: &mut state_impl,
701 last_binlog_offset: &mut last_binlog_offset,
702 is_snapshot_paused: &mut is_snapshot_paused,
703 rate_limit_rps: &mut self.rate_limit_rps,
704 };
705 #[for_await]
706 for msg in upstream.by_ref() {
707 match msg? {
708 Message::Chunk(chunk) if handler.cursor.is_none() => {
709 if chunk.has_visible_rows()
712 && let Some(chunk_offset) =
713 get_cdc_chunk_last_offset(handler.offset_parse_func, &chunk)?
714 && handler
715 .last_binlog_offset
716 .as_ref()
717 .is_none_or(|last| *last < chunk_offset)
718 {
719 *handler.last_binlog_offset = Some(chunk_offset);
720 }
721 }
722 msg => match handler.handle_upstream(msg).await? {
723 NoSnapshotUpstreamOutcome::Continue(Some(msg)) => {
724 let is_barrier = matches!(msg, Message::Barrier(_));
725
726 yield msg;
727
728 if is_barrier {
729 break;
730 }
731 }
732 NoSnapshotUpstreamOutcome::Continue(None) => {}
733 NoSnapshotUpstreamOutcome::Stop(barrier) => {
734 yield Message::Barrier(barrier);
735
736 return Ok(());
737 }
738 },
739 }
740 }
741
742 tracing::info!(%table_id,
743 upstream_table_name,
744 initial_binlog_offset = ?last_binlog_offset,
745 ?current_pk_pos,
746 is_snapshot_paused,
747 "start cdc backfill loop");
748
749 let mut upstream_chunk_buffer: Vec<StreamChunk> = vec![];
751 let mut needs_reader_rebuild = false;
752
753 'backfill_loop: loop {
754 if needs_reader_rebuild {
755 needs_reader_rebuild = false;
756 tracing::info!(
757 %table_id,
758 upstream_table_name,
759 "rebuilding CDC table reader after snapshot read failure"
760 );
761
762 if let Err(error) = upstream_table_reader.disconnect().await {
763 tracing::warn!(
764 error = %error.as_report(),
765 %table_id,
766 upstream_table_name,
767 "failed to disconnect CDC snapshot reader; continuing with rebuild"
768 );
769 }
770
771 let mut future = Box::pin(create_table_reader_with_retry(
772 self.external_table.clone(),
773 self.actor_ctx.id,
774 self.actor_ctx.fragment_id,
775 ));
776
777 let mut handler = NoSnapshotUpstreamHandler {
778 actor_ctx: &self.actor_ctx,
779 metrics: &self.metrics,
780 table_id,
781 upstream_table_name: &upstream_table_name,
782 offset_parse_func: &mut offset_parse_func,
783 output_indices: &self.output_indices,
784 pk_indices: &pk_indices,
785 pk_order: &pk_order,
786 cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
787 pk_pos,
788 pk_needs_unsigned_i64_compare: &pk_needs_unsigned_i64_compare,
789 }),
790 total_snapshot_row_count,
791 state_impl: &mut state_impl,
792 last_binlog_offset: &mut last_binlog_offset,
793 is_snapshot_paused: &mut is_snapshot_paused,
794 rate_limit_rps: &mut self.rate_limit_rps,
795 };
796 let table_reader = loop {
797 match handler.wait_for_reader(&mut upstream, &mut future).await? {
798 Either::Left(NoSnapshotUpstreamOutcome::Continue(msg)) => {
799 if let Some(msg) = msg {
800 yield msg;
801 }
802 }
803 Either::Left(NoSnapshotUpstreamOutcome::Stop(barrier)) => {
804 yield Message::Barrier(barrier);
805
806 return Ok(());
807 }
808 Either::Right(table_reader) => break table_reader,
809 }
810 };
811
812 upstream_table_reader =
813 UpstreamTableReader::new(self.external_table.clone(), table_reader);
814 tracing::info!(
815 %table_id,
816 upstream_table_name,
817 "CDC table reader rebuilt successfully"
818 );
819 }
820
821 let mut should_bypass_snapshot_stream_patch =
822 self.rate_limit_rps.is_some_and(|val| val == 0);
823 let left_upstream = upstream.by_ref().map(Either::Left);
824
825 let mut snapshot_read_row_cnt: usize = 0;
826 let read_args = SnapshotReadArgs::new(
827 current_pk_pos.clone(),
828 self.rate_limit_rps,
829 pk_indices.clone(),
830 additional_columns.clone(),
831 schema_table_name.clone(),
832 external_database_name.clone(),
833 );
834 let right_snapshot = pin!(
835 upstream_table_reader
836 .snapshot_read_full_table(read_args, self.options.snapshot_batch_size)
837 .map(Either::Right)
838 );
839 let (right_snapshot, snapshot_valve) = pausable(right_snapshot);
840 if is_snapshot_paused {
841 snapshot_valve.pause();
842 }
843
844 let mut backfill_stream =
846 select_with_strategy(left_upstream, right_snapshot, |_: &mut ()| {
847 stream::PollNext::Left
848 });
849
850 let mut cur_barrier_snapshot_processed_rows: u64 = 0;
851 let mut cur_barrier_upstream_processed_rows: u64 = 0;
852 let mut barrier_count: u32 = 0;
853 let mut pending_barrier = None;
854
855 #[for_await]
856 for either in &mut backfill_stream {
857 match either {
858 Either::Left(msg) => {
860 match msg? {
861 Message::Barrier(barrier) => {
862 barrier_count += 1;
864 let mut needs_rebuild_snapshot =
865 barrier_count == self.options.snapshot_barrier_interval;
866
867 if let Some(mutation) = barrier.mutation.as_deref() {
868 match mutation {
869 Mutation::Pause => {
870 is_snapshot_paused = true;
871 snapshot_valve.pause();
872 }
873 Mutation::Resume => {
874 is_snapshot_paused = false;
875 snapshot_valve.resume();
876 }
877 Mutation::Throttle(_) => {
878 if let Some(entry) = mutation
879 .backfill_throttle_config(
880 self.actor_ctx.fragment_id,
881 )
882 && entry.rate_limit != self.rate_limit_rps
883 {
884 should_bypass_snapshot_stream_patch = self
887 .rate_limit_rps
888 .is_some_and(|val| val == 0);
889 self.rate_limit_rps = entry.rate_limit;
890 needs_rebuild_snapshot = true;
891 }
892 }
893 mutation if mutation.is_stop(self.actor_ctx.id) => {
894 tracing::info!(
896 %table_id,
897 upstream_table_name,
898 "CdcBackfill has been dropped due to config change"
899 );
900 state_impl.commit_state(barrier.epoch).await?;
901 yield Message::Barrier(barrier);
902 break 'backfill_loop;
903 }
904 _ => (),
905 }
906 }
907
908 if needs_rebuild_snapshot || needs_reader_rebuild {
911 pending_barrier = Some(barrier);
913 tracing::debug!(
914 %table_id,
915 ?current_pk_pos,
916 ?snapshot_read_row_cnt,
917 "Prepare to start a new snapshot"
918 );
919 break;
921 } else {
922 let (
924 emitted_upstream_chunks,
925 consumed_upstream_row_count,
926 consumed_binlog_offset,
927 ) = Self::consume_upstream_chunk_buffer(
928 &offset_parse_func,
929 &mut upstream_chunk_buffer,
930 current_pk_pos.as_ref(),
931 PkCompareInfo {
932 indices: &pk_indices,
933 order: &pk_order,
934 needs_unsigned_i64_compare:
935 &pk_needs_unsigned_i64_compare,
936 },
937 &last_binlog_offset,
938 &self.output_indices,
939 )?;
940 cur_barrier_upstream_processed_rows +=
941 consumed_upstream_row_count;
942 if let Some(consumed_binlog_offset) = consumed_binlog_offset
943 {
944 last_binlog_offset = Some(consumed_binlog_offset);
945 }
946 for chunk in emitted_upstream_chunks {
947 yield Message::Chunk(chunk);
948 }
949
950 Self::report_metrics(
951 &self.metrics,
952 cur_barrier_snapshot_processed_rows,
953 cur_barrier_upstream_processed_rows,
954 );
955
956 state_impl
958 .mutate_and_commit_state(
959 current_pk_pos.clone(),
960 last_binlog_offset.clone(),
961 total_snapshot_row_count,
962 false,
963 barrier.epoch,
964 )
965 .await?;
966
967 yield Message::Barrier(barrier);
969 }
970 }
971 Message::Chunk(chunk) => {
972 if chunk.cardinality() == 0 {
974 continue;
975 }
976
977 let chunk_binlog_offset =
978 get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
979
980 tracing::trace!(
981 "recv changelog chunk: chunk_offset {:?}, capactiy {}",
982 chunk_binlog_offset,
983 chunk.capacity()
984 );
985
986 if let (Some(last_binlog_offset), Some(chunk_offset)) =
990 (last_binlog_offset.as_ref(), chunk_binlog_offset)
991 && chunk_offset < *last_binlog_offset
992 {
993 tracing::trace!(
994 "skip changelog chunk: chunk_offset {:?}, capacity {}",
995 chunk_offset,
996 chunk.capacity()
997 );
998 continue;
999 }
1000
1001 upstream_chunk_buffer.push(chunk.compact_vis());
1003 }
1004 Message::Watermark(_) => {
1005 }
1007 }
1008 }
1009 Either::Right(msg) => {
1011 match msg {
1012 Ok(None) => {
1013 tracing::info!(
1014 %table_id,
1015 ?last_binlog_offset,
1016 ?current_pk_pos,
1017 "snapshot read stream ends"
1018 );
1019 for chunk in upstream_chunk_buffer.drain(..) {
1024 yield Message::Chunk(mapping_chunk(
1025 chunk,
1026 &self.output_indices,
1027 ));
1028 }
1029
1030 break 'backfill_loop;
1032 }
1033 Ok(Some(chunk)) => {
1034 current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
1038
1039 tracing::trace!(
1040 "got a snapshot chunk: len {}, current_pk_pos {:?}",
1041 chunk.cardinality(),
1042 current_pk_pos
1043 );
1044 let chunk_cardinality = chunk.cardinality() as u64;
1045 cur_barrier_snapshot_processed_rows += chunk_cardinality;
1046 total_snapshot_row_count += chunk_cardinality;
1047 yield Message::Chunk(mapping_chunk(
1048 chunk,
1049 &self.output_indices,
1050 ));
1051 }
1052 Err(error) => {
1053 needs_reader_rebuild = true;
1054 tracing::warn!(
1055 error = %error.as_report(),
1056 %table_id,
1057 upstream_table_name,
1058 "failed to read CDC snapshot; rebuilding reader after a barrier"
1059 );
1060 }
1061 }
1062 }
1063 }
1064 }
1065
1066 assert!(pending_barrier.is_some(), "pending_barrier must exist");
1067 let pending_barrier = pending_barrier.unwrap();
1068
1069 let (_, mut snapshot_stream) = backfill_stream.into_inner();
1075 if !should_bypass_snapshot_stream_patch && is_snapshot_paused {
1077 snapshot_valve.resume();
1078 }
1079 if !should_bypass_snapshot_stream_patch
1080 && let Some(msg) = snapshot_stream
1081 .next()
1082 .instrument_await("consume_snapshot_stream_once")
1083 .await
1084 {
1085 let Either::Right(msg) = msg else {
1086 bail!("BUG: snapshot_read contains upstream messages");
1087 };
1088 match msg {
1089 Ok(None) => {
1090 tracing::info!(
1091 %table_id,
1092 ?last_binlog_offset,
1093 ?current_pk_pos,
1094 "snapshot read stream ends in the force emit branch"
1095 );
1096 for chunk in upstream_chunk_buffer.drain(..) {
1099 yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
1100 }
1101
1102 state_impl
1104 .mutate_and_commit_state(
1105 current_pk_pos.clone(),
1106 last_binlog_offset.clone(),
1107 total_snapshot_row_count,
1108 true,
1109 pending_barrier.epoch,
1110 )
1111 .await?;
1112
1113 yield Message::Barrier(pending_barrier);
1114 break 'backfill_loop;
1116 }
1117 Ok(Some(_)) if is_snapshot_paused => {
1118 }
1120 Ok(Some(chunk)) => {
1121 current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
1123
1124 let row_count = chunk.cardinality() as u64;
1125 cur_barrier_snapshot_processed_rows += row_count;
1126 total_snapshot_row_count += row_count;
1127 snapshot_read_row_cnt += row_count as usize;
1128
1129 tracing::debug!(
1130 %table_id,
1131 ?current_pk_pos,
1132 ?snapshot_read_row_cnt,
1133 "force emit a snapshot chunk"
1134 );
1135 yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
1136 }
1137 Err(error) => {
1138 needs_reader_rebuild = true;
1139 tracing::warn!(
1140 error = %error.as_report(),
1141 %table_id,
1142 upstream_table_name,
1143 "failed to read CDC snapshot; rebuilding reader after a barrier"
1144 );
1145 }
1146 }
1147 }
1148
1149 let mut consumed_binlog_offset = None;
1152
1153 if let Some(current_pos) = ¤t_pk_pos {
1154 for chunk in upstream_chunk_buffer.drain(..) {
1155 cur_barrier_upstream_processed_rows += chunk.cardinality() as u64;
1156
1157 consumed_binlog_offset =
1160 get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
1161
1162 yield Message::Chunk(mapping_chunk(
1163 mark_cdc_chunk(
1164 &offset_parse_func,
1165 chunk,
1166 current_pos,
1167 &pk_indices,
1168 &pk_order,
1169 &pk_needs_unsigned_i64_compare,
1170 last_binlog_offset.clone(),
1171 )?,
1172 &self.output_indices,
1173 ));
1174 }
1175 } else {
1176 upstream_chunk_buffer.clear();
1179 }
1180
1181 if consumed_binlog_offset.is_some() {
1183 last_binlog_offset.clone_from(&consumed_binlog_offset);
1184 }
1185
1186 Self::report_metrics(
1187 &self.metrics,
1188 cur_barrier_snapshot_processed_rows,
1189 cur_barrier_upstream_processed_rows,
1190 );
1191
1192 state_impl
1194 .mutate_and_commit_state(
1195 current_pk_pos.clone(),
1196 last_binlog_offset.clone(),
1197 total_snapshot_row_count,
1198 false,
1199 pending_barrier.epoch,
1200 )
1201 .await?;
1202
1203 yield Message::Barrier(pending_barrier);
1204 }
1205 upstream_table_reader.disconnect().await?;
1206 } else if self.options.disable_backfill {
1207 tracing::info!(
1209 %table_id,
1210 upstream_table_name,
1211 "CdcBackfill has been disabled"
1212 );
1213 state_impl
1214 .mutate_state(
1215 current_pk_pos.clone(),
1216 last_binlog_offset.clone(),
1217 total_snapshot_row_count,
1218 true,
1219 )
1220 .await?;
1221 }
1222
1223 tracing::info!(
1224 %table_id,
1225 upstream_table_name,
1226 "CdcBackfill has already finished and will forward messages directly to the downstream"
1227 );
1228
1229 while let Some(Ok(msg)) = upstream.next().await {
1232 if let Some(msg) = mapping_message(msg, &self.output_indices) {
1233 if let Message::Barrier(barrier) = &msg {
1235 state_impl
1237 .mutate_and_commit_state(
1238 current_pk_pos.clone(),
1239 last_binlog_offset.clone(),
1240 total_snapshot_row_count,
1241 true,
1242 barrier.epoch,
1243 )
1244 .await?;
1245
1246 if let Some(progress) = self.progress.as_mut() {
1248 progress.finish(barrier.epoch, total_snapshot_row_count);
1249 }
1250 yield msg;
1251 break;
1253 }
1254 yield msg;
1255 }
1256 }
1257
1258 #[for_await]
1262 for msg in upstream {
1263 if let Some(msg) = mapping_message(msg?, &self.output_indices) {
1266 if let Message::Barrier(barrier) = &msg {
1267 state_impl.commit_state(barrier.epoch).await?;
1269 }
1270 yield msg;
1271 }
1272 }
1273 }
1274}
1275
1276#[try_stream(ok = Message, error = StreamExecutorError)]
1277pub async fn transform_upstream(
1278 upstream: BoxedMessageStream,
1279 output_columns: Vec<ColumnDesc>,
1280 properties: BTreeMap<String, String>,
1281 handle_postgres_toast_columns: bool,
1282) {
1283 let (timestamp_handling, timestamptz_handling, time_handling, bigint_unsigned_handling) =
1284 get_cdc_json_parse_handling_from_properties(&properties);
1285 let props = SpecificParserConfig {
1286 encoding_config: EncodingProperties::Json(JsonProperties {
1287 use_schema_registry: false,
1288 timestamp_handling,
1289 timestamptz_handling,
1290 time_handling,
1291 bigint_unsigned_handling,
1292 handle_postgres_toast_columns,
1293 }),
1294 protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()),
1296 };
1297
1298 let columns_with_meta = output_columns
1300 .iter()
1301 .map(SourceColumnDesc::from)
1302 .collect_vec();
1303 let mut parser = DebeziumParser::new(
1304 props,
1305 columns_with_meta.clone(),
1306 Arc::new(SourceContext::dummy()),
1307 properties.cdc_source_type().debezium_numeric_handling(),
1308 )
1309 .await
1310 .map_err(StreamExecutorError::connector_error)?;
1311
1312 pin_mut!(upstream);
1313 #[for_await]
1314 for msg in upstream {
1315 let mut msg = msg?;
1316 if let Message::Chunk(chunk) = &mut msg {
1317 let parsed_chunk = parse_debezium_chunk(&mut parser, chunk).await?;
1318 let _ = std::mem::replace(chunk, parsed_chunk);
1319 }
1320 yield msg;
1321 }
1322}
1323
1324async fn parse_debezium_chunk(
1325 parser: &mut DebeziumParser,
1326 chunk: &StreamChunk,
1327) -> StreamExecutorResult<StreamChunk> {
1328 let mut builder = SourceStreamChunkBuilder::new(
1335 parser.columns().to_vec(),
1336 SourceCtrlOpts {
1337 chunk_size: chunk.capacity(),
1338 split_txn: false,
1339 },
1340 );
1341
1342 let payloads = chunk.data_chunk().project(&[0]);
1346 let offsets = chunk.data_chunk().project(&[1]).compact_vis();
1347
1348 for payload in payloads.rows() {
1350 let ScalarRefImpl::Jsonb(jsonb_ref) = payload.datum_at(0).expect("payload must exist")
1351 else {
1352 panic!("payload must be jsonb");
1353 };
1354
1355 parser
1356 .parse_inner(
1357 None,
1358 Some(jsonb_ref.to_string().as_bytes().to_vec()),
1359 builder.row_writer(),
1360 )
1361 .await
1362 .unwrap();
1363 }
1364 builder.finish_current_chunk();
1365
1366 let parsed_chunk = {
1367 let mut iter = builder.consume_ready_chunks();
1368 assert_eq!(1, iter.len());
1369 iter.next().unwrap()
1370 };
1371 assert_eq!(parsed_chunk.capacity(), chunk.capacity()); let (ops, mut columns, vis) = parsed_chunk.into_inner();
1373 columns.extend(offsets.into_parts().0);
1377
1378 Ok(StreamChunk::from_parts(
1379 ops,
1380 DataChunk::from_parts(columns.into(), vis),
1381 ))
1382}
1383
1384impl<S: StateStore> Execute for CdcBackfillExecutor<S> {
1385 fn execute(self: Box<Self>) -> BoxedMessageStream {
1386 self.execute_inner().boxed()
1387 }
1388}
1389
1390#[cfg(test)]
1391mod tests {
1392 use std::collections::BTreeMap;
1393 use std::str::FromStr;
1394 use std::time::Duration;
1395
1396 use either::Either;
1397 use futures::{StreamExt, pin_mut, stream};
1398 use risingwave_common::array::{Array, DataChunk, Op, StreamChunk};
1399 use risingwave_common::catalog::{
1400 CdcKeyComparison, ColumnDesc, ColumnId, Field, Schema, TableId,
1401 };
1402 use risingwave_common::row::{OwnedRow, Row};
1403 use risingwave_common::types::{DataType, Datum, JsonbVal, ScalarImpl};
1404 use risingwave_common::util::epoch::test_epoch;
1405 use risingwave_common::util::iter_util::ZipEqFast;
1406 use risingwave_common::util::sort_util::OrderType;
1407 use risingwave_connector::source::cdc::CdcScanOptions;
1408 use risingwave_connector::source::cdc::external::mock_external_table::MockExternalTableReader;
1409 use risingwave_connector::source::cdc::external::mysql::MysqlOffset;
1410 use risingwave_connector::source::cdc::external::{
1411 CdcOffset, ExternalCdcTableType, ExternalTableConfig, ExternalTableReaderImpl,
1412 SchemaTableName,
1413 };
1414 use risingwave_storage::memory::MemoryStateStore;
1415
1416 use super::{
1417 PkCompareInfo, build_reader_and_poll_upstream, can_poll_upstream_while_creating_reader,
1418 };
1419 use crate::common::table::test_utils::gen_pbtable;
1420 use crate::executor::backfill::cdc::cdc_backfill::transform_upstream;
1421 use crate::executor::backfill::cdc::state::CdcBackfillState;
1422 use crate::executor::monitor::StreamingMetrics;
1423 use crate::executor::prelude::StateTable;
1424 use crate::executor::test_utils::{MessageSender, MockSource};
1425 use crate::executor::{
1426 ActorContext, Barrier, CdcBackfillExecutor, ExternalStorageTable, Message,
1427 };
1428
1429 #[tokio::test]
1430 async fn test_transform_upstream_chunk() {
1431 let schema = Schema::new(vec![
1432 Field::unnamed(DataType::Jsonb), Field::unnamed(DataType::Varchar), Field::unnamed(DataType::Varchar), ]);
1436 let stream_key = vec![1];
1437 let (mut tx, source) = MockSource::channel();
1438 let source = source.into_executor(schema.clone(), stream_key.clone());
1439 let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#;
1441
1442 let datums: Vec<Datum> = vec![
1443 Some(JsonbVal::from_str(payload).unwrap().into()),
1444 Some("file: 1.binlog, pos: 100".to_owned().into()),
1445 Some("mydb.orders".to_owned().into()),
1446 ];
1447
1448 let mut builders = schema.create_array_builders(8);
1449 for (builder, datum) in builders.iter_mut().zip_eq_fast(datums.iter()) {
1450 builder.append(datum.clone());
1451 }
1452 let columns = builders
1453 .into_iter()
1454 .map(|builder| builder.finish().into())
1455 .collect();
1456
1457 let chunk = StreamChunk::from_parts(vec![Op::Insert], DataChunk::new(columns, 1));
1459
1460 tx.push_chunk(chunk);
1461 let upstream = Box::new(source).execute();
1462
1463 let columns = vec![
1465 ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
1466 ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
1467 ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
1468 ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
1469 ColumnDesc::named("O_ORDERDATE", ColumnId::new(5), DataType::Date),
1470 ColumnDesc::named("commit_ts", ColumnId::new(6), DataType::Timestamptz),
1471 ];
1472
1473 let parsed_stream = transform_upstream(upstream, columns, BTreeMap::new(), false);
1474 pin_mut!(parsed_stream);
1475 let message = parsed_stream
1476 .next()
1477 .await
1478 .expect("transform stream should yield the input chunk")
1479 .expect("transforming the CDC chunk should succeed");
1480 let Message::Chunk(chunk) = message else {
1481 panic!("expected a transformed chunk");
1482 };
1483 assert_eq!(
1484 chunk
1485 .rows()
1486 .map(|(op, row)| (op, row.to_owned_row()))
1487 .collect::<Vec<_>>(),
1488 vec![(
1489 Op::Insert,
1490 OwnedRow::new(vec![
1491 Some(ScalarImpl::Int64(5)),
1492 Some(ScalarImpl::Int64(44485)),
1493 Some(ScalarImpl::Utf8("F".into())),
1494 Some(ScalarImpl::Decimal("144659.20".parse().unwrap())),
1495 Some(ScalarImpl::Date("1994-07-30".parse().unwrap())),
1496 None,
1497 Some(ScalarImpl::Utf8("file: 1.binlog, pos: 100".into())),
1498 ]),
1499 )]
1500 );
1501 }
1502
1503 #[tokio::test]
1504 async fn test_finished_backfill_forwards_without_reader() {
1505 let actor_context = ActorContext::for_test(1);
1506 let table_id = TableId::new(1234);
1507 let external_storage_table = ExternalStorageTable::new(
1508 table_id,
1509 SchemaTableName {
1510 schema_name: "public".to_owned(),
1511 table_name: "missing_table".to_owned(),
1512 },
1513 "mydb".to_owned(),
1514 ExternalTableConfig::default(),
1515 ExternalCdcTableType::Unspecified,
1516 Schema::new(vec![Field::with_name(DataType::Int64, "O_ORDERKEY")]),
1517 vec![OrderType::ascending()],
1518 Some(vec![CdcKeyComparison::Native]),
1519 vec![0],
1520 );
1521 let schema = Schema::new(vec![
1522 Field::unnamed(DataType::Jsonb), Field::unnamed(DataType::Varchar), Field::unnamed(DataType::Varchar), ]);
1526 let stream_key = vec![1];
1527 let (mut tx, source) = MockSource::channel();
1528 let source = source
1529 .stop_on_finish(false)
1530 .into_executor(schema.clone(), stream_key.clone());
1531 let output_indices = vec![1, 0, 4]; let output_columns = vec![
1533 ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
1534 ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
1535 ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
1536 ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
1537 ColumnDesc::named("O_DUMMY", ColumnId::new(5), DataType::Int64),
1538 ColumnDesc::named("commit_ts", ColumnId::new(6), DataType::Timestamptz),
1539 ];
1540 let store = MemoryStateStore::new();
1541 let mut persisted_state =
1542 CdcBackfillState::new(table_id, create_cdc_state_table(store.clone()).await, 5);
1543 persisted_state
1544 .init_epoch(Barrier::new_test_barrier(test_epoch(6)).epoch)
1545 .await
1546 .unwrap();
1547 persisted_state
1548 .mutate_and_commit_state(
1549 Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
1550 Some(CdcOffset::Mysql(MysqlOffset::new(
1551 "1.binlog".to_owned(),
1552 100,
1553 ))),
1554 1,
1555 true,
1556 Barrier::new_test_barrier(test_epoch(7)).epoch,
1557 )
1558 .await
1559 .unwrap();
1560 let state_table = create_cdc_state_table(store).await;
1561 let cdc = CdcBackfillExecutor::new(
1562 actor_context,
1563 external_storage_table,
1564 source,
1565 output_indices,
1566 output_columns,
1567 None,
1568 StreamingMetrics::unused().into(),
1569 state_table,
1570 None,
1571 CdcScanOptions::default(),
1572 BTreeMap::default(),
1573 );
1574 let s = cdc.execute_inner();
1575 pin_mut!(s);
1576
1577 tx.send_barrier(Barrier::new_test_barrier(test_epoch(8)));
1579 {
1581 let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_DUMMY": 100 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#;
1582 let datums: Vec<Datum> = vec![
1583 Some(JsonbVal::from_str(payload).unwrap().into()),
1584 Some("file: 1.binlog, pos: 100".to_owned().into()),
1585 Some("mydb.orders".to_owned().into()),
1586 ];
1587 let mut builders = schema.create_array_builders(8);
1588 for (builder, datum) in builders.iter_mut().zip_eq_fast(datums.iter()) {
1589 builder.append(datum.clone());
1590 }
1591 let columns = builders
1592 .into_iter()
1593 .map(|builder| builder.finish().into())
1594 .collect();
1595 let chunk = StreamChunk::from_parts(vec![Op::Insert], DataChunk::new(columns, 1));
1597
1598 tx.push_chunk(chunk);
1599 }
1600 let _first_barrier = s.next().await.unwrap();
1601 let upstream_change_log = s.next().await.unwrap().unwrap();
1602 let Message::Chunk(chunk) = upstream_change_log else {
1603 panic!("expect chunk");
1604 };
1605 assert_eq!(chunk.columns().len(), 3);
1606 assert_eq!(chunk.rows().count(), 1);
1607 assert_eq!(
1608 chunk.columns()[0].as_int64().iter().collect::<Vec<_>>(),
1609 vec![Some(44485)]
1610 );
1611 assert_eq!(
1612 chunk.columns()[1].as_int64().iter().collect::<Vec<_>>(),
1613 vec![Some(5)]
1614 );
1615 assert_eq!(
1616 chunk.columns()[2].as_int64().iter().collect::<Vec<_>>(),
1617 vec![Some(100)]
1618 );
1619
1620 drop(tx);
1621 assert!(s.next().await.is_none());
1622 }
1623
1624 #[test]
1625 fn test_reader_startup_polling_selection() {
1626 let recovered_position = OwnedRow::new(vec![Some(ScalarImpl::Int64(1))]);
1627
1628 assert!(!can_poll_upstream_while_creating_reader(
1629 Some(&recovered_position),
1630 false,
1631 ));
1632 assert!(can_poll_upstream_while_creating_reader(None, false));
1633 assert!(can_poll_upstream_while_creating_reader(
1634 Some(&recovered_position),
1635 true,
1636 ));
1637 }
1638
1639 #[tokio::test]
1640 async fn test_snapshot_read_error_rebuilds_reader() {
1641 let external_table = create_mock_external_table().with_mock_snapshot_errors([1, 0]);
1642 let reader_create_probe = external_table.clone();
1643 let (tx, executor) = create_mock_cdc_backfill(
1644 external_table,
1645 create_cdc_state_table(MemoryStateStore::new()).await,
1646 CdcScanOptions {
1647 snapshot_barrier_interval: 1,
1648 ..Default::default()
1649 },
1650 );
1651 let executor = executor.execute_inner();
1652 pin_mut!(executor);
1653
1654 tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
1655 assert!(matches!(
1656 executor.next().await.unwrap().unwrap(),
1657 Message::Barrier(_)
1658 ));
1659
1660 tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
1662 assert!(matches!(
1663 executor.next().await.unwrap().unwrap(),
1664 Message::Barrier(_)
1665 ));
1666
1667 assert!(
1668 tokio::time::timeout(Duration::from_millis(20), executor.next())
1669 .await
1670 .is_err()
1671 );
1672 assert_eq!(reader_create_probe.mock_reader_create_count(), 1);
1673
1674 tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
1676 assert!(matches!(
1677 executor.next().await.unwrap().unwrap(),
1678 Message::Barrier(_)
1679 ));
1680
1681 let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
1683 panic!("replacement reader should produce a snapshot chunk");
1684 };
1685 assert_eq!(chunk.cardinality(), 6);
1686 assert_eq!(reader_create_probe.mock_reader_create_count(), 2);
1687
1688 tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
1689 assert!(matches!(
1690 executor.next().await.unwrap().unwrap(),
1691 Message::Barrier(_)
1692 ));
1693 }
1694
1695 #[tokio::test]
1696 async fn test_snapshot_read_error_before_first_chunk_survives_recovery() {
1697 let scan_options = || CdcScanOptions {
1698 snapshot_barrier_interval: 1,
1699 ..Default::default()
1700 };
1701
1702 let memory_state_store = MemoryStateStore::new();
1703
1704 {
1707 let (tx, executor) = create_mock_cdc_backfill(
1708 create_mock_external_table().with_mock_snapshot_errors([1]),
1709 create_cdc_state_table(memory_state_store.clone()).await,
1710 scan_options(),
1711 );
1712 let executor = executor.execute_inner();
1713
1714 pin_mut!(executor);
1715
1716 for epoch in 1..=3 {
1717 tx.send_barrier(Barrier::new_test_barrier(test_epoch(epoch)));
1718 assert!(matches!(
1719 executor.next().await.unwrap().unwrap(),
1720 Message::Barrier(_)
1721 ));
1722 }
1723 }
1724
1725 let mut restored_state = CdcBackfillState::new(
1726 TableId::new(1234),
1727 create_cdc_state_table(memory_state_store.clone()).await,
1728 5,
1729 );
1730 restored_state
1731 .init_epoch(Barrier::new_test_barrier(test_epoch(4)).epoch)
1732 .await
1733 .unwrap();
1734
1735 let state = restored_state.restore_state().await.unwrap();
1737 assert!(!state.is_finished);
1738 assert_eq!(state.current_pk_pos, None);
1739 assert_eq!(state.last_cdc_offset, None);
1740 assert_eq!(state.row_count, 0);
1741
1742 let (tx, executor) = create_mock_cdc_backfill(
1745 create_mock_external_table(),
1746 create_cdc_state_table(memory_state_store).await,
1747 scan_options(),
1748 );
1749 let executor = executor.execute_inner();
1750
1751 pin_mut!(executor);
1752
1753 for epoch in 4..=5 {
1754 tx.send_barrier(Barrier::new_test_barrier(test_epoch(epoch)));
1755 assert!(matches!(
1756 executor.next().await.unwrap().unwrap(),
1757 Message::Barrier(_)
1758 ));
1759 }
1760 let Message::Chunk(chunk) = tokio::time::timeout(Duration::from_secs(5), executor.next())
1761 .await
1762 .expect("recovered backfill should read the snapshot")
1763 .unwrap()
1764 .unwrap()
1765 else {
1766 panic!("recovered reader should produce a snapshot chunk");
1767 };
1768
1769 assert_eq!(chunk.cardinality(), 6);
1770 }
1771
1772 #[tokio::test]
1773 async fn test_disabled_cdc_backfill_finishes_without_table_reader() {
1774 let actor_context = ActorContext::for_test(1);
1775 let external_storage_table = ExternalStorageTable::new(
1776 TableId::new(1234),
1777 SchemaTableName {
1778 schema_name: "public".to_owned(),
1779 table_name: "orders".to_owned(),
1780 },
1781 "db".to_owned(),
1782 ExternalTableConfig::default(),
1783 ExternalCdcTableType::Unspecified,
1784 Schema::new(vec![Field::with_name(DataType::Int64, "id")]),
1785 vec![OrderType::ascending()],
1786 Some(vec![CdcKeyComparison::Native]),
1787 vec![0],
1788 );
1789 let schema = Schema::new(vec![
1790 Field::unnamed(DataType::Jsonb),
1791 Field::unnamed(DataType::Varchar),
1792 Field::unnamed(DataType::Varchar),
1793 ]);
1794 let stream_key = vec![1];
1795 let (tx, source) = MockSource::channel();
1796 let source = source.into_executor(schema, stream_key);
1797 let output_columns = vec![ColumnDesc::named("id", ColumnId::new(1), DataType::Int64)];
1798 let memory_state_store = MemoryStateStore::new();
1799 let state_table = create_cdc_state_table(memory_state_store.clone()).await;
1800 let cdc = CdcBackfillExecutor::new(
1801 actor_context,
1802 external_storage_table,
1803 source,
1804 vec![0],
1805 output_columns,
1806 None,
1807 StreamingMetrics::unused().into(),
1808 state_table,
1809 None,
1810 CdcScanOptions {
1811 disable_backfill: true,
1812 ..CdcScanOptions::default()
1813 },
1814 BTreeMap::default(),
1815 );
1816 let executor = cdc.execute_inner();
1817 pin_mut!(executor);
1818
1819 tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
1820 assert!(matches!(
1821 executor.next().await.unwrap().unwrap(),
1822 Message::Barrier(_)
1823 ));
1824
1825 tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
1826 assert!(matches!(
1827 executor.next().await.unwrap().unwrap(),
1828 Message::Barrier(_)
1829 ));
1830
1831 let mut restored_state = CdcBackfillState::new(
1832 TableId::new(1234),
1833 create_cdc_state_table(memory_state_store).await,
1834 5,
1835 );
1836 restored_state
1837 .init_epoch(Barrier::new_test_barrier(test_epoch(2)).epoch)
1838 .await
1839 .unwrap();
1840 let state = restored_state.restore_state().await.unwrap();
1841 assert!(state.is_finished);
1842 assert_eq!(state.last_cdc_offset, None);
1843 }
1844
1845 #[tokio::test]
1846 async fn test_unfinished_cdc_backfill_rejects_null_offset_on_restore() {
1847 let memory_state_store = MemoryStateStore::new();
1848 let mut state_writer = CdcBackfillState::new(
1849 TableId::new(1234),
1850 create_cdc_state_table(memory_state_store.clone()).await,
1851 5,
1852 );
1853 state_writer
1854 .init_epoch(Barrier::new_test_barrier(test_epoch(1)).epoch)
1855 .await
1856 .unwrap();
1857 state_writer
1858 .mutate_and_commit_state(
1859 Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(10))])),
1860 None,
1861 10,
1862 false,
1863 Barrier::new_test_barrier(test_epoch(2)).epoch,
1864 )
1865 .await
1866 .unwrap();
1867
1868 let mut restored_state = CdcBackfillState::new(
1869 TableId::new(1234),
1870 create_cdc_state_table(memory_state_store).await,
1871 5,
1872 );
1873 restored_state
1874 .init_epoch(Barrier::new_test_barrier(test_epoch(2)).epoch)
1875 .await
1876 .unwrap();
1877
1878 assert!(restored_state.restore_state().await.is_err());
1879 }
1880
1881 fn create_raw_cdc_chunk(rows: &[(&str, &str)]) -> StreamChunk {
1882 let schema = Schema::new(vec![
1883 Field::unnamed(DataType::Jsonb),
1884 Field::unnamed(DataType::Varchar),
1885 ]);
1886 let mut builders = schema.create_array_builders(rows.len());
1887 for (payload, offset) in rows {
1888 let payload_datum: Datum = Some(JsonbVal::from_str(payload).unwrap().into());
1889 let offset_datum: Datum = Some((*offset).into());
1890 builders[0].append(payload_datum);
1891 builders[1].append(offset_datum);
1892 }
1893 let columns = builders
1894 .into_iter()
1895 .map(|builder| builder.finish().into())
1896 .collect();
1897 StreamChunk::from_parts(
1898 vec![Op::Insert; rows.len()],
1899 DataChunk::new(columns, rows.len()),
1900 )
1901 }
1902
1903 fn create_mock_external_table() -> ExternalStorageTable {
1904 ExternalStorageTable::new(
1905 TableId::new(1234),
1906 SchemaTableName {
1907 schema_name: "public".to_owned(),
1908 table_name: "mock_table".to_owned(),
1909 },
1910 "mydb".to_owned(),
1911 ExternalTableConfig::default(),
1912 ExternalCdcTableType::Mock,
1913 Schema::new(vec![
1914 Field::with_name(DataType::Int64, "id"),
1915 Field::with_name(DataType::Float64, "price"),
1916 ]),
1917 vec![OrderType::ascending()],
1918 Some(vec![CdcKeyComparison::Native]),
1919 vec![0],
1920 )
1921 }
1922
1923 fn create_mock_cdc_backfill(
1926 external_table: ExternalStorageTable,
1927 state_table: StateTable<MemoryStateStore>,
1928 scan_options: CdcScanOptions,
1929 ) -> (MessageSender, CdcBackfillExecutor<MemoryStateStore>) {
1930 let (tx, source) = MockSource::channel();
1931 let source = source.into_executor(
1932 Schema::new(vec![
1933 Field::unnamed(DataType::Jsonb),
1934 Field::unnamed(DataType::Varchar),
1935 ]),
1936 vec![0],
1937 );
1938 let executor = CdcBackfillExecutor::new(
1939 ActorContext::for_test(0x1a),
1940 external_table,
1941 source,
1942 vec![0, 1],
1943 vec![
1944 ColumnDesc::named("id", ColumnId::new(1), DataType::Int64),
1945 ColumnDesc::named("price", ColumnId::new(2), DataType::Float64),
1946 ],
1947 None,
1948 StreamingMetrics::unused().into(),
1949 state_table,
1950 None,
1951 scan_options,
1952 BTreeMap::default(),
1953 );
1954 (tx, executor)
1955 }
1956
1957 async fn create_cdc_state_table(store: MemoryStateStore) -> StateTable<MemoryStateStore> {
1958 let state_schema = Schema::new(vec![
1959 Field::with_name(DataType::Varchar, "split_id"),
1960 Field::with_name(DataType::Int64, "id"),
1961 Field::with_name(DataType::Boolean, "backfill_finished"),
1962 Field::with_name(DataType::Int64, "row_count"),
1963 Field::with_name(DataType::Jsonb, "cdc_offset"),
1964 ]);
1965 let column_descs = vec![
1966 ColumnDesc::unnamed(ColumnId::from(0), state_schema[0].data_type.clone()),
1967 ColumnDesc::unnamed(ColumnId::from(1), state_schema[1].data_type.clone()),
1968 ColumnDesc::unnamed(ColumnId::from(2), state_schema[2].data_type.clone()),
1969 ColumnDesc::unnamed(ColumnId::from(3), state_schema[3].data_type.clone()),
1970 ColumnDesc::unnamed(ColumnId::from(4), state_schema[4].data_type.clone()),
1971 ];
1972
1973 StateTable::from_table_catalog(
1974 &gen_pbtable(
1975 TableId::from(0x42),
1976 column_descs,
1977 vec![OrderType::ascending()],
1978 vec![0],
1979 0,
1980 ),
1981 store,
1982 None,
1983 )
1984 .await
1985 }
1986
1987 async fn create_recovering_cdc_backfill() -> (
1988 MessageSender,
1989 CdcBackfillExecutor<MemoryStateStore>,
1990 MemoryStateStore,
1991 ) {
1992 let memory_state_store = MemoryStateStore::new();
1993 let mut state_writer = CdcBackfillState::new(
1994 TableId::new(1234),
1995 create_cdc_state_table(memory_state_store.clone()).await,
1996 5,
1997 );
1998 state_writer
1999 .init_epoch(Barrier::new_test_barrier(test_epoch(1)).epoch)
2000 .await
2001 .unwrap();
2002 state_writer
2003 .mutate_and_commit_state(
2004 Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2005 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2006 5,
2007 false,
2008 Barrier::new_test_barrier(test_epoch(2)).epoch,
2009 )
2010 .await
2011 .unwrap();
2012
2013 let (tx, executor) = create_mock_cdc_backfill(
2014 create_mock_external_table(),
2015 create_cdc_state_table(memory_state_store.clone()).await,
2016 CdcScanOptions::default(),
2017 );
2018 (tx, executor, memory_state_store)
2019 }
2020
2021 async fn assert_recovery_offset(
2022 memory_state_store: MemoryStateStore,
2023 expected_position: u64,
2024 epoch: u64,
2025 ) {
2026 let mut restored_state = CdcBackfillState::new(
2027 TableId::new(1234),
2028 create_cdc_state_table(memory_state_store).await,
2029 5,
2030 );
2031 restored_state
2032 .init_epoch(Barrier::new_test_barrier(epoch).epoch)
2033 .await
2034 .unwrap();
2035 let state = restored_state.restore_state().await.unwrap();
2036 assert_eq!(
2037 state.last_cdc_offset,
2038 Some(CdcOffset::Mysql(MysqlOffset::new(
2039 "1.binlog".to_owned(),
2040 expected_position,
2041 )))
2042 );
2043 }
2044
2045 #[tokio::test]
2046 async fn test_recovery_filters_upstream_while_reader_is_pending() {
2047 let chunk = StreamChunk::from_rows(
2048 &[
2049 (
2050 Op::Insert,
2051 OwnedRow::new(vec![
2052 Some(ScalarImpl::Int64(4)),
2053 Some(ScalarImpl::Float64(44.04.into())),
2054 Some(ScalarImpl::Utf8(
2055 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2056 .into(),
2057 )),
2058 ]),
2059 ),
2060 (
2061 Op::Insert,
2062 OwnedRow::new(vec![
2063 Some(ScalarImpl::Int64(6)),
2064 Some(ScalarImpl::Float64(66.06.into())),
2065 Some(ScalarImpl::Utf8(
2066 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2067 .into(),
2068 )),
2069 ]),
2070 ),
2071 ],
2072 &[DataType::Int64, DataType::Float64, DataType::Varchar],
2073 );
2074 let mut upstream = stream::iter([
2075 Ok(Message::Chunk(chunk)),
2076 Ok(Message::Barrier(Barrier::new_test_barrier(test_epoch(4)))),
2077 ]);
2078 let mut reader_future = Box::pin(std::future::pending::<ExternalTableReaderImpl>());
2079
2080 let Either::Left(Message::Chunk(chunk)) =
2081 build_reader_and_poll_upstream(&mut upstream, &mut reader_future)
2082 .await
2083 .unwrap()
2084 else {
2085 panic!("expected an upstream chunk while the reader is pending");
2086 };
2087
2088 let parser = ExternalCdcTableType::Mock.get_cdc_offset_parser().unwrap();
2089 let (chunk, consumed_offset) =
2090 CdcBackfillExecutor::<MemoryStateStore>::filter_recovery_chunk(
2091 &parser,
2092 chunk,
2093 &OwnedRow::new(vec![Some(ScalarImpl::Int64(5))]),
2094 PkCompareInfo {
2095 indices: &[0],
2096 order: &[OrderType::ascending()],
2097 needs_unsigned_i64_compare: &[false],
2098 },
2099 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2100 &[0, 1],
2101 )
2102 .unwrap();
2103 let chunk = chunk.expect("the scanned row should be emitted");
2104 assert_eq!(chunk.cardinality(), 1);
2105 assert_eq!(
2106 chunk.row_at(0).1.to_owned_row()[0],
2107 Some(ScalarImpl::Int64(4))
2108 );
2109 assert_eq!(
2110 consumed_offset,
2111 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 4)))
2112 );
2113 assert!(matches!(
2114 build_reader_and_poll_upstream(&mut upstream, &mut reader_future)
2115 .await
2116 .unwrap(),
2117 Either::Left(Message::Barrier(_))
2118 ));
2119 }
2120
2121 #[tokio::test]
2122 async fn test_recovery_filters_updates_before_backfill_barrier() {
2123 let (mut tx, executor, memory_state_store) = create_recovering_cdc_backfill().await;
2124 let executor = executor.execute_inner();
2125 pin_mut!(executor);
2126
2127 tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2128 assert!(matches!(
2129 executor.next().await.unwrap().unwrap(),
2130 Message::Barrier(_)
2131 ));
2132
2133 tx.push_chunk(create_raw_cdc_chunk(&[
2134 (
2135 r#"{ "payload": { "before": { "id": 1, "price": 11.01 }, "after": { "id": 1, "price": 12.01 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "u", "ts_ms": 1695277757017, "transaction": null } }"#,
2136 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2137 ),
2138 (
2139 r#"{ "payload": { "before": { "id": 6, "price": 66.06 }, "after": { "id": 6, "price": 67.06 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "u", "ts_ms": 1695277757017, "transaction": null } }"#,
2140 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2141 ),
2142 ]));
2143 tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2144
2145 let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
2146 panic!("expected the recovered update before the barrier");
2147 };
2148 let rows = chunk
2149 .rows()
2150 .map(|(op, row)| (op, row.to_owned_row()))
2151 .collect::<Vec<_>>();
2152 assert_eq!(rows.len(), 1);
2153 assert_eq!(rows[0].0, Op::Insert);
2154 assert_eq!(rows[0].1[0], Some(ScalarImpl::Int64(1)));
2155 assert!(matches!(
2156 executor.next().await.unwrap().unwrap(),
2157 Message::Barrier(_)
2158 ));
2159 assert_recovery_offset(memory_state_store, 4, test_epoch(4)).await;
2160 }
2161
2162 #[tokio::test]
2163 async fn test_recovery_preserves_scanned_inserts_and_deletes() {
2164 let (mut tx, executor, _) = create_recovering_cdc_backfill().await;
2165 let executor = executor.execute_inner();
2166 pin_mut!(executor);
2167
2168 tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2169 assert!(matches!(
2170 executor.next().await.unwrap().unwrap(),
2171 Message::Barrier(_)
2172 ));
2173 tx.push_chunk(create_raw_cdc_chunk(&[(
2175 r#"{ "payload": { "before": null, "after": { "id": 3, "price": 30.01 }, "op": "c" } }"#,
2176 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":1},"isHeartbeat":false}"#,
2177 )]));
2178 tx.push_chunk(create_raw_cdc_chunk(&[
2179 (
2180 r#"{ "payload": { "before": null, "after": { "id": 3, "price": 31.01 }, "op": "c" } }"#,
2181 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":1},"isHeartbeat":false}"#,
2182 ),
2183 (
2184 r#"{ "payload": { "before": null, "after": { "id": 4, "price": 44.04 }, "op": "c" } }"#,
2185 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":2},"isHeartbeat":false}"#,
2186 ),
2187 (
2188 r#"{ "payload": { "before": { "id": 5, "price": 55.05 }, "after": null, "op": "d" } }"#,
2189 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2190 ),
2191 ]));
2192 tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2193
2194 let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
2195 panic!("expected the scanned-prefix changes before the barrier");
2196 };
2197 let rows = chunk
2198 .rows()
2199 .map(|(op, row)| (op, row.to_owned_row()))
2200 .collect::<Vec<_>>();
2201 assert_eq!(rows.len(), 2);
2202 assert_eq!(rows[0].0, Op::Insert);
2203 assert_eq!(rows[0].1[0], Some(ScalarImpl::Int64(4)));
2204 assert_eq!(rows[1].0, Op::Delete);
2205 assert_eq!(rows[1].1[0], Some(ScalarImpl::Int64(5)));
2206 assert!(matches!(
2207 executor.next().await.unwrap().unwrap(),
2208 Message::Barrier(_)
2209 ));
2210 }
2211
2212 #[tokio::test]
2213 async fn test_recovery_does_not_leave_deleted_unscanned_row() {
2214 let (mut tx, mut executor, memory_state_store) = create_recovering_cdc_backfill().await;
2215 executor.options.snapshot_barrier_interval = 1;
2216 let executor = executor.execute_inner();
2217 pin_mut!(executor);
2218
2219 tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2220 assert!(matches!(
2221 executor.next().await.unwrap().unwrap(),
2222 Message::Barrier(_)
2223 ));
2224
2225 tx.push_chunk(create_raw_cdc_chunk(&[(
2227 r#"{ "payload": { "before": null, "after": { "id": 100, "price": 100.01 }, "op": "c" } }"#,
2228 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2229 )]));
2230 tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2231
2232 tx.push_chunk(create_raw_cdc_chunk(&[(
2235 r#"{ "payload": { "before": { "id": 100, "price": 100.01 }, "after": null, "op": "d" } }"#,
2236 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2237 )]));
2238 tx.send_barrier(Barrier::new_test_barrier(test_epoch(5)));
2239 tx.send_barrier(Barrier::new_test_barrier(test_epoch(6)));
2241
2242 let mut materialized = BTreeMap::new();
2243 loop {
2244 match executor.next().await.unwrap().unwrap() {
2245 Message::Chunk(chunk) => {
2246 for (op, row) in chunk.rows() {
2247 let pk = row.datum_at(0).unwrap().into_int64();
2248 match op {
2249 Op::Insert => {
2250 materialized.insert(pk, row.to_owned_row());
2251 }
2252 Op::Delete => {
2253 materialized.remove(&pk);
2254 }
2255 _ => unreachable!("CDC updates are converted to inserts"),
2256 }
2257 }
2258 }
2259 Message::Barrier(barrier) if barrier.epoch.curr == test_epoch(6) => break,
2260 Message::Barrier(_) => {}
2261 Message::Watermark(_) => panic!("unexpected watermark during backfill"),
2262 }
2263 }
2264
2265 assert!(!materialized.contains_key(&100));
2266 let mut restored_state = CdcBackfillState::new(
2267 TableId::new(1234),
2268 create_cdc_state_table(memory_state_store).await,
2269 5,
2270 );
2271 restored_state
2272 .init_epoch(Barrier::new_test_barrier(test_epoch(6)).epoch)
2273 .await
2274 .unwrap();
2275 assert!(restored_state.restore_state().await.unwrap().is_finished);
2276 }
2277
2278 #[test]
2279 fn test_consume_upstream_chunk_buffer_retains_future_rows() {
2280 let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2281 &[
2282 (
2283 Op::Insert,
2284 OwnedRow::new(vec![
2285 Some(ScalarImpl::Int64(1)),
2286 Some(ScalarImpl::Int64(100)),
2287 Some(ScalarImpl::Utf8(
2288 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2289 .into(),
2290 )),
2291 ]),
2292 ),
2293 (
2294 Op::Insert,
2295 OwnedRow::new(vec![
2296 Some(ScalarImpl::Int64(6)),
2297 Some(ScalarImpl::Int64(600)),
2298 Some(ScalarImpl::Utf8(
2299 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2300 .into(),
2301 )),
2302 ]),
2303 ),
2304 ],
2305 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2306 )];
2307
2308 let (emitted_chunks, drained_row_count, drained_offset) =
2309 CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2310 &MockExternalTableReader::get_cdc_offset_parser(),
2311 &mut upstream_chunk_buffer,
2312 Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2313 PkCompareInfo {
2314 indices: &[0],
2315 order: &[OrderType::ascending()],
2316 needs_unsigned_i64_compare: &[false],
2317 },
2318 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2319 &[0, 1],
2320 )
2321 .unwrap();
2322
2323 assert_eq!(drained_row_count, 1);
2324 assert_eq!(
2325 drained_offset,
2326 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2327 );
2328 assert_eq!(emitted_chunks.len(), 1);
2329 assert_eq!(emitted_chunks[0].rows().count(), 1);
2330 assert_eq!(
2331 emitted_chunks[0].rows().next().unwrap().1.to_owned_row(),
2332 OwnedRow::new(vec![
2333 Some(ScalarImpl::Int64(1)),
2334 Some(ScalarImpl::Int64(100))
2335 ])
2336 );
2337
2338 assert_eq!(upstream_chunk_buffer.len(), 1);
2339 assert_eq!(upstream_chunk_buffer[0].rows().count(), 1);
2340 assert_eq!(
2341 upstream_chunk_buffer[0]
2342 .rows()
2343 .next()
2344 .unwrap()
2345 .1
2346 .to_owned_row(),
2347 OwnedRow::new(vec![
2348 Some(ScalarImpl::Int64(6)),
2349 Some(ScalarImpl::Int64(600)),
2350 Some(ScalarImpl::Utf8(
2351 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2352 .into(),
2353 )),
2354 ])
2355 );
2356 }
2357
2358 #[test]
2359 fn test_consume_buffer_uses_unsigned_bigint_pk_order() {
2360 let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2361 &[
2362 (
2363 Op::Insert,
2364 OwnedRow::new(vec![
2365 Some(ScalarImpl::Int64(4)),
2366 Some(ScalarImpl::Int64(400)),
2367 Some(ScalarImpl::Utf8(
2368 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2369 .into(),
2370 )),
2371 ]),
2372 ),
2373 (
2374 Op::Insert,
2375 OwnedRow::new(vec![
2376 Some(ScalarImpl::Int64(-1)),
2378 Some(ScalarImpl::Int64(900)),
2379 Some(ScalarImpl::Utf8(
2380 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2381 .into(),
2382 )),
2383 ]),
2384 ),
2385 ],
2386 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2387 )];
2388
2389 let (emitted_chunks, drained_row_count, drained_offset) =
2390 CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2391 &MockExternalTableReader::get_cdc_offset_parser(),
2392 &mut upstream_chunk_buffer,
2393 Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2394 PkCompareInfo {
2395 indices: &[0],
2396 order: &[OrderType::ascending()],
2397 needs_unsigned_i64_compare: &[true],
2398 },
2399 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2400 &[0, 1],
2401 )
2402 .unwrap();
2403
2404 assert_eq!(drained_row_count, 1);
2405 assert_eq!(
2406 drained_offset,
2407 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2408 );
2409 assert_eq!(emitted_chunks.len(), 1);
2410 assert_eq!(
2411 emitted_chunks[0].rows().next().unwrap().1.to_owned_row(),
2412 OwnedRow::new(vec![
2413 Some(ScalarImpl::Int64(4)),
2414 Some(ScalarImpl::Int64(400))
2415 ])
2416 );
2417
2418 assert_eq!(upstream_chunk_buffer.len(), 1);
2419 assert_eq!(
2420 upstream_chunk_buffer[0]
2421 .rows()
2422 .next()
2423 .unwrap()
2424 .1
2425 .to_owned_row(),
2426 OwnedRow::new(vec![
2427 Some(ScalarImpl::Int64(-1)),
2428 Some(ScalarImpl::Int64(900)),
2429 Some(ScalarImpl::Utf8(
2430 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2431 .into(),
2432 )),
2433 ])
2434 );
2435 }
2436
2437 #[test]
2438 fn test_consume_buffer_non_monotonic_pk_in_chunk() {
2439 let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2440 &[
2441 (
2442 Op::Insert,
2443 OwnedRow::new(vec![
2444 Some(ScalarImpl::Int64(1)),
2445 Some(ScalarImpl::Int64(100)),
2446 Some(ScalarImpl::Utf8(
2447 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2448 .into(),
2449 )),
2450 ]),
2451 ),
2452 (
2453 Op::Insert,
2454 OwnedRow::new(vec![
2455 Some(ScalarImpl::Int64(6)),
2456 Some(ScalarImpl::Int64(600)),
2457 Some(ScalarImpl::Utf8(
2458 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2459 .into(),
2460 )),
2461 ]),
2462 ),
2463 (
2464 Op::Insert,
2465 OwnedRow::new(vec![
2466 Some(ScalarImpl::Int64(2)),
2467 Some(ScalarImpl::Int64(200)),
2468 Some(ScalarImpl::Utf8(
2469 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2470 .into(),
2471 )),
2472 ]),
2473 ),
2474 ],
2475 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2476 )];
2477
2478 let (emitted_chunks, drained_row_count, drained_offset) =
2479 CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2480 &MockExternalTableReader::get_cdc_offset_parser(),
2481 &mut upstream_chunk_buffer,
2482 Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2483 PkCompareInfo {
2484 indices: &[0],
2485 order: &[OrderType::ascending()],
2486 needs_unsigned_i64_compare: &[false],
2487 },
2488 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2489 &[0, 1],
2490 )
2491 .unwrap();
2492
2493 assert_eq!(drained_row_count, 2);
2494 assert_eq!(
2495 drained_offset,
2496 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2497 );
2498 assert_eq!(emitted_chunks.len(), 1);
2499 assert_eq!(emitted_chunks[0].rows().count(), 2);
2500 assert_eq!(
2501 emitted_chunks[0]
2502 .rows()
2503 .map(|(_, row)| row.to_owned_row())
2504 .collect::<Vec<_>>(),
2505 vec![
2506 OwnedRow::new(vec![
2507 Some(ScalarImpl::Int64(1)),
2508 Some(ScalarImpl::Int64(100))
2509 ]),
2510 OwnedRow::new(vec![
2511 Some(ScalarImpl::Int64(2)),
2512 Some(ScalarImpl::Int64(200))
2513 ]),
2514 ]
2515 );
2516
2517 assert_eq!(upstream_chunk_buffer.len(), 1);
2518 assert_eq!(upstream_chunk_buffer[0].rows().count(), 1);
2519 assert_eq!(
2520 upstream_chunk_buffer[0]
2521 .rows()
2522 .next()
2523 .unwrap()
2524 .1
2525 .to_owned_row(),
2526 OwnedRow::new(vec![
2527 Some(ScalarImpl::Int64(6)),
2528 Some(ScalarImpl::Int64(600)),
2529 Some(ScalarImpl::Utf8(
2530 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2531 .into(),
2532 )),
2533 ])
2534 );
2535 }
2536
2537 #[test]
2538 fn test_consume_buffer_processes_chunks_after_future_row() {
2539 let mut upstream_chunk_buffer = vec![
2540 StreamChunk::from_rows(
2541 &[
2542 (
2543 Op::Insert,
2544 OwnedRow::new(vec![
2545 Some(ScalarImpl::Int64(1)),
2546 Some(ScalarImpl::Int64(100)),
2547 Some(ScalarImpl::Utf8(
2548 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2549 .into(),
2550 )),
2551 ]),
2552 ),
2553 (
2554 Op::Insert,
2555 OwnedRow::new(vec![
2556 Some(ScalarImpl::Int64(6)),
2557 Some(ScalarImpl::Int64(600)),
2558 Some(ScalarImpl::Utf8(
2559 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2560 .into(),
2561 )),
2562 ]),
2563 ),
2564 ],
2565 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2566 ),
2567 StreamChunk::from_rows(
2568 &[(
2569 Op::Insert,
2570 OwnedRow::new(vec![
2571 Some(ScalarImpl::Int64(2)),
2572 Some(ScalarImpl::Int64(200)),
2573 Some(ScalarImpl::Utf8(
2574 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2575 .into(),
2576 )),
2577 ]),
2578 )],
2579 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2580 ),
2581 ];
2582
2583 let (emitted_chunks, drained_row_count, drained_offset) =
2584 CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2585 &MockExternalTableReader::get_cdc_offset_parser(),
2586 &mut upstream_chunk_buffer,
2587 Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2588 PkCompareInfo {
2589 indices: &[0],
2590 order: &[OrderType::ascending()],
2591 needs_unsigned_i64_compare: &[false],
2592 },
2593 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2594 &[0, 1],
2595 )
2596 .unwrap();
2597
2598 assert_eq!(drained_row_count, 2);
2599 assert_eq!(
2600 drained_offset,
2601 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2602 );
2603 assert_eq!(emitted_chunks.len(), 2);
2604 assert_eq!(emitted_chunks[0].rows().count(), 1);
2605 assert_eq!(emitted_chunks[1].rows().count(), 1);
2606 assert_eq!(
2607 emitted_chunks[1].rows().next().unwrap().1.to_owned_row(),
2608 OwnedRow::new(vec![
2609 Some(ScalarImpl::Int64(2)),
2610 Some(ScalarImpl::Int64(200))
2611 ])
2612 );
2613
2614 assert_eq!(upstream_chunk_buffer.len(), 1);
2615 assert_eq!(
2616 upstream_chunk_buffer[0]
2617 .rows()
2618 .next()
2619 .unwrap()
2620 .1
2621 .to_owned_row(),
2622 OwnedRow::new(vec![
2623 Some(ScalarImpl::Int64(6)),
2624 Some(ScalarImpl::Int64(600)),
2625 Some(ScalarImpl::Utf8(
2626 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2627 .into(),
2628 )),
2629 ])
2630 );
2631 }
2632
2633 #[test]
2634 fn test_consume_buffer_advances_offset_after_preceding_rows_are_emitted() {
2635 let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2636 &[
2637 (
2638 Op::Insert,
2639 OwnedRow::new(vec![
2640 Some(ScalarImpl::Int64(6)),
2641 Some(ScalarImpl::Int64(600)),
2642 Some(ScalarImpl::Utf8(
2643 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2644 .into(),
2645 )),
2646 ]),
2647 ),
2648 (
2649 Op::Insert,
2650 OwnedRow::new(vec![
2651 Some(ScalarImpl::Int64(2)),
2652 Some(ScalarImpl::Int64(200)),
2653 Some(ScalarImpl::Utf8(
2654 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2655 .into(),
2656 )),
2657 ]),
2658 ),
2659 ],
2660 &[DataType::Int64, DataType::Int64, DataType::Varchar],
2661 )];
2662
2663 let (_, drained_row_count, drained_offset) =
2664 CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2665 &MockExternalTableReader::get_cdc_offset_parser(),
2666 &mut upstream_chunk_buffer,
2667 Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(6))])),
2668 PkCompareInfo {
2669 indices: &[0],
2670 order: &[OrderType::ascending()],
2671 needs_unsigned_i64_compare: &[false],
2672 },
2673 &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3))),
2674 &[0, 1],
2675 )
2676 .unwrap();
2677
2678 assert_eq!(drained_row_count, 2);
2679 assert_eq!(
2680 drained_offset,
2681 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 5)))
2682 );
2683 assert!(upstream_chunk_buffer.is_empty());
2684 }
2685
2686 #[tokio::test]
2687 async fn test_cdc_backfill_persists_buffered_offset_on_checkpoint() {
2688 let memory_state_store = MemoryStateStore::new();
2689 let state_table = create_cdc_state_table(memory_state_store.clone()).await;
2690
2691 let (mut tx, executor) = create_mock_cdc_backfill(
2692 create_mock_external_table(),
2693 state_table,
2694 CdcScanOptions {
2695 snapshot_barrier_interval: 10,
2696 ..Default::default()
2697 },
2698 );
2699 let executor = executor.execute_inner();
2700 pin_mut!(executor);
2701
2702 tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
2703 assert!(matches!(
2704 executor.next().await.unwrap().unwrap(),
2705 Message::Barrier(_)
2706 ));
2707
2708 tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
2709 assert!(matches!(
2710 executor.next().await.unwrap().unwrap(),
2711 Message::Barrier(_)
2712 ));
2713
2714 assert!(matches!(
2715 executor.next().await.unwrap().unwrap(),
2716 Message::Chunk(_)
2717 ));
2718
2719 tx.push_chunk(create_raw_cdc_chunk(&[
2720 (
2721 r#"{ "payload": { "before": null, "after": { "id": 1, "price": 10.01 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
2722 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2723 ),
2724 (
2725 r#"{ "payload": { "before": null, "after": { "id": 6, "price": 66.06 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
2726 r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2727 ),
2728 ]));
2729 tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2730
2731 assert!(matches!(
2732 executor.next().await.unwrap().unwrap(),
2733 Message::Chunk(_)
2734 ));
2735 assert!(matches!(
2736 executor.next().await.unwrap().unwrap(),
2737 Message::Barrier(_)
2738 ));
2739
2740 let mut restored_state = CdcBackfillState::new(
2741 TableId::new(1234),
2742 create_cdc_state_table(memory_state_store).await,
2743 5,
2744 );
2745 restored_state
2746 .init_epoch(Barrier::new_test_barrier(test_epoch(3)).epoch)
2747 .await
2748 .unwrap();
2749 let state = restored_state.restore_state().await.unwrap();
2750 assert_eq!(
2751 state.last_cdc_offset,
2752 Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 4)))
2753 );
2754 }
2755}