Skip to main content

risingwave_stream/executor/backfill/cdc/
cdc_backfill.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::BTreeMap;
16use std::pin::Pin;
17
18use either::Either;
19use futures::stream;
20use futures::stream::select_with_strategy;
21use itertools::Itertools;
22use risingwave_common::array::{DataChunk, Op};
23use risingwave_common::bail;
24use risingwave_common::bitmap::BitmapBuilder;
25use risingwave_common::catalog::{CdcKeyComparison, ColumnDesc};
26use risingwave_common::row::RowExt;
27use risingwave_common::util::sort_util::OrderType;
28use risingwave_connector::WithPropertiesExt;
29use risingwave_connector::parser::{
30    BigintUnsignedHandlingMode, ByteStreamSourceParser, DebeziumParser, DebeziumProps,
31    EncodingProperties, JsonProperties, ProtocolProperties, SourceStreamChunkBuilder,
32    SpecificParserConfig, TimeHandling, TimestampHandling, TimestamptzHandling,
33};
34use risingwave_connector::source::cdc::CdcScanOptions;
35use risingwave_connector::source::cdc::external::{
36    CdcOffset, CdcOffsetParseFunc, ExternalCdcTableType, ExternalTableReaderImpl,
37};
38use risingwave_connector::source::{SourceColumnDesc, SourceContext, SourceCtrlOpts};
39use rw_futures_util::pausable;
40use thiserror_ext::AsReport;
41
42use crate::executor::backfill::cdc::state::CdcBackfillState;
43use crate::executor::backfill::cdc::upstream_table::external::ExternalStorageTable;
44use crate::executor::backfill::cdc::upstream_table::reader::{
45    build_reader_and_poll_upstream, create_table_reader_with_retry,
46};
47use crate::executor::backfill::cdc::upstream_table::snapshot::{
48    SnapshotReadArgs, UpstreamTableRead, UpstreamTableReader,
49};
50use crate::executor::backfill::utils::{
51    cmp_pk_unsigned_aware, get_cdc_chunk_last_offset, get_new_pos, mapping_chunk, mapping_message,
52    mark_cdc_chunk,
53};
54use crate::executor::monitor::CdcBackfillMetrics;
55use crate::executor::prelude::*;
56use crate::executor::{Mutation, TableId};
57use crate::task::CreateMviewProgressReporter;
58
59/// `split_id`, `is_finished`, `row_count`, `cdc_offset` all occupy 1 column each.
60const METADATA_STATE_LEN: usize = 4;
61
62struct PkCompareInfo<'a> {
63    indices: &'a [usize],
64    order: &'a [OrderType],
65    needs_unsigned_i64_compare: &'a [bool],
66}
67
68fn can_poll_upstream_while_creating_reader(
69    current_pk_pos: Option<&OwnedRow>,
70    pk_comparisons_are_known: bool,
71) -> bool {
72    current_pk_pos.is_none() || pk_comparisons_are_known
73}
74
75/// Result of handling an upstream message while no snapshot is running.
76enum NoSnapshotUpstreamOutcome {
77    /// Forward the message downstream, if any, and keep going.
78    /// `None` means nothing to emit: a watermark, or a chunk with no visible rows to emit.
79    Continue(Option<Message>),
80    /// The actor is stopped: forward the barrier and exit the executor.
81    Stop(Barrier),
82}
83
84/// Snapshot cursor together with the PK comparisons needed to filter upstream changes
85/// against it.
86#[derive(Clone, Copy)]
87struct SnapshotCursor<'a> {
88    pk_pos: &'a OwnedRow,
89    pk_needs_unsigned_i64_compare: &'a [bool],
90}
91
92struct NoSnapshotUpstreamHandler<'a, S: StateStore> {
93    actor_ctx: &'a ActorContextRef,
94    metrics: &'a CdcBackfillMetrics,
95    table_id: TableId,
96    upstream_table_name: &'a str,
97    offset_parse_func: &'a mut CdcOffsetParseFunc,
98    output_indices: &'a [usize],
99    pk_indices: &'a [usize],
100    pk_order: &'a [OrderType],
101    /// `None` means the next snapshot starts from the beginning.
102    cursor: Option<SnapshotCursor<'a>>,
103    total_snapshot_row_count: u64,
104    state_impl: &'a mut CdcBackfillState<S>,
105    last_binlog_offset: &'a mut Option<CdcOffset>,
106    is_snapshot_paused: &'a mut bool,
107    rate_limit_rps: &'a mut Option<u32>,
108}
109
110impl<S: StateStore> NoSnapshotUpstreamHandler<'_, S> {
111    async fn handle_upstream(
112        &mut self,
113        msg: Message,
114    ) -> StreamExecutorResult<NoSnapshotUpstreamOutcome> {
115        match msg {
116            Message::Barrier(barrier) => {
117                if let Some(mutation) = barrier.mutation.as_deref() {
118                    match mutation {
119                        Mutation::Pause => {
120                            *self.is_snapshot_paused = true;
121                            tracing::info!(
122                                table_id = %self.table_id,
123                                upstream_table_name = self.upstream_table_name,
124                                "snapshot is paused by barrier"
125                            );
126                        }
127                        Mutation::Resume => {
128                            *self.is_snapshot_paused = false;
129                            tracing::info!(
130                                table_id = %self.table_id,
131                                upstream_table_name = self.upstream_table_name,
132                                "snapshot is resumed by barrier"
133                            );
134                        }
135                        Mutation::Throttle(_) => {
136                            if let Some(entry) =
137                                mutation.backfill_throttle_config(self.actor_ctx.fragment_id)
138                            {
139                                *self.rate_limit_rps = entry.rate_limit;
140                            }
141                        }
142                        mutation if mutation.is_stop(self.actor_ctx.id) => {
143                            tracing::info!(
144                                table_id = %self.table_id,
145                                upstream_table_name = self.upstream_table_name,
146                                "CdcBackfill has been dropped due to config change"
147                            );
148                            self.state_impl.commit_state(barrier.epoch).await?;
149
150                            return Ok(NoSnapshotUpstreamOutcome::Stop(barrier));
151                        }
152                        _ => (),
153                    }
154                }
155
156                // Commit after all preceding upstream chunks have been emitted.
157                self.state_impl
158                    .mutate_and_commit_state(
159                        self.cursor.map(|cursor| cursor.pk_pos.clone()),
160                        self.last_binlog_offset.clone(),
161                        self.total_snapshot_row_count,
162                        false,
163                        barrier.epoch,
164                    )
165                    .await?;
166
167                Ok(NoSnapshotUpstreamOutcome::Continue(Some(Message::Barrier(
168                    barrier,
169                ))))
170            }
171            Message::Chunk(chunk) => {
172                let Some(cursor) = self.cursor else {
173                    // The next snapshot starts from the beginning and covers these changes.
174                    return Ok(NoSnapshotUpstreamOutcome::Continue(None));
175                };
176
177                if !chunk.has_visible_rows() {
178                    return Ok(NoSnapshotUpstreamOutcome::Continue(None));
179                }
180
181                let (chunk, consumed_offset) = CdcBackfillExecutor::<S>::filter_recovery_chunk(
182                    self.offset_parse_func,
183                    chunk,
184                    cursor.pk_pos,
185                    PkCompareInfo {
186                        indices: self.pk_indices,
187                        order: self.pk_order,
188                        needs_unsigned_i64_compare: cursor.pk_needs_unsigned_i64_compare,
189                    },
190                    self.last_binlog_offset,
191                    self.output_indices,
192                )?;
193
194                if let Some(consumed_offset) = consumed_offset {
195                    *self.last_binlog_offset = Some(consumed_offset);
196                }
197
198                let chunk = chunk.map(|chunk| {
199                    CdcBackfillExecutor::<S>::report_metrics(
200                        self.metrics,
201                        0,
202                        chunk.cardinality() as u64,
203                    );
204
205                    Message::Chunk(chunk)
206                });
207
208                Ok(NoSnapshotUpstreamOutcome::Continue(chunk))
209            }
210            Message::Watermark(_) => Ok(NoSnapshotUpstreamOutcome::Continue(None)),
211        }
212    }
213
214    async fn wait_for_reader(
215        &mut self,
216        upstream: &mut (impl Stream<Item = StreamExecutorResult<Message>> + Unpin),
217        future: &mut Pin<Box<impl Future<Output = ExternalTableReaderImpl>>>,
218    ) -> StreamExecutorResult<Either<NoSnapshotUpstreamOutcome, ExternalTableReaderImpl>> {
219        match build_reader_and_poll_upstream(upstream, future).await? {
220            Either::Left(msg) => Ok(Either::Left(self.handle_upstream(msg).await?)),
221            Either::Right(table_reader) => Ok(Either::Right(table_reader)),
222        }
223    }
224}
225
226// The TimestampHandling/TimestamptzHandling/TimeHandling parser's behavior depends on the debezium.time.precision.mode setting:
227// - If left unset, Debezium defaults to time.precision.mode=microseconds (per debezium.properties), and the parser uses Micro.
228// - If set to "connect", Debezium uses time.precision.mode=connect, and the parser uses Milli by design.
229// - If set to any other value, Debezium applies that specific value, and the default parser is used to maintain backward compatibility.
230pub(crate) fn get_cdc_json_parse_handling_from_properties(
231    properties: &BTreeMap<String, String>,
232) -> (
233    Option<TimestampHandling>,
234    Option<TimestamptzHandling>,
235    Option<TimeHandling>,
236    Option<BigintUnsignedHandlingMode>,
237) {
238    let (timestamp_handling, timestamptz_handling, time_handling) = match properties
239        .get("debezium.time.precision.mode")
240    {
241        None => (
242            Some(TimestampHandling::Micro),
243            Some(TimestamptzHandling::Micro),
244            Some(TimeHandling::Micro),
245        ),
246        Some(m) if m == "connect" => (
247            Some(TimestampHandling::Milli),
248            Some(TimestamptzHandling::Milli),
249            Some(TimeHandling::Milli),
250        ),
251        Some(other) => {
252            // backward compatibility.
253            tracing::warn!(
254                "Unsupported debezium.time.precision.mode = {other}, fall back to default parser."
255            );
256            (None, None, None)
257        }
258    };
259    let bigint_unsigned_handling = properties
260        .get("debezium.bigint.unsigned.handling.mode")
261        .is_some_and(|v| v == "precise")
262        .then_some(BigintUnsignedHandlingMode::Precise);
263    (
264        timestamp_handling,
265        timestamptz_handling,
266        time_handling,
267        bigint_unsigned_handling,
268    )
269}
270
271pub struct CdcBackfillExecutor<S: StateStore> {
272    actor_ctx: ActorContextRef,
273
274    /// The external table to be backfilled
275    external_table: ExternalStorageTable,
276
277    /// Upstream changelog stream which may contain metadata columns, e.g. `_rw_offset`
278    upstream: Executor,
279
280    /// The column indices need to be forwarded to the downstream from the upstream and table scan.
281    output_indices: Vec<usize>,
282
283    /// The schema of output chunk, including additional columns if any
284    output_columns: Vec<ColumnDesc>,
285
286    /// State table of the `CdcBackfill` executor
287    state_impl: CdcBackfillState<S>,
288
289    // TODO: introduce a CdcBackfillProgress to report finish to Meta
290    // This object is just a stub right now
291    progress: Option<CreateMviewProgressReporter>,
292
293    metrics: CdcBackfillMetrics,
294
295    /// Rate limit in rows/s.
296    rate_limit_rps: Option<u32>,
297
298    options: CdcScanOptions,
299
300    properties: BTreeMap<String, String>,
301}
302
303impl<S: StateStore> CdcBackfillExecutor<S> {
304    #[expect(clippy::too_many_arguments)]
305    pub fn new(
306        actor_ctx: ActorContextRef,
307        external_table: ExternalStorageTable,
308        upstream: Executor,
309        output_indices: Vec<usize>,
310        output_columns: Vec<ColumnDesc>,
311        progress: Option<CreateMviewProgressReporter>,
312        metrics: Arc<StreamingMetrics>,
313        state_table: StateTable<S>,
314        rate_limit_rps: Option<u32>,
315        options: CdcScanOptions,
316        properties: BTreeMap<String, String>,
317    ) -> Self {
318        let pk_indices = external_table.pk_indices();
319        let upstream_table_id = external_table.table_id();
320        let state_impl = CdcBackfillState::new(
321            upstream_table_id,
322            state_table,
323            pk_indices.len() + METADATA_STATE_LEN,
324        );
325
326        let metrics = metrics.new_cdc_backfill_metrics(external_table.table_id(), actor_ctx.id);
327        Self {
328            actor_ctx,
329            external_table,
330            upstream,
331            output_indices,
332            output_columns,
333            state_impl,
334            progress,
335            metrics,
336            rate_limit_rps,
337            options,
338            properties,
339        }
340    }
341
342    fn report_metrics(
343        metrics: &CdcBackfillMetrics,
344        snapshot_processed_row_count: u64,
345        upstream_processed_row_count: u64,
346    ) {
347        metrics
348            .cdc_backfill_snapshot_read_row_count
349            .inc_by(snapshot_processed_row_count);
350
351        metrics
352            .cdc_backfill_upstream_output_row_count
353            .inc_by(upstream_processed_row_count);
354    }
355
356    /// Partitions buffered CDC rows into emitted and retained chunk views.
357    ///
358    /// Rows in the active binlog range with `PK <= current_pk_pos` are visible in
359    /// the emitted chunks. Rows with `PK > current_pk_pos` are visible in the
360    /// retained buffer.
361    fn consume_upstream_chunk_buffer(
362        offset_parse_func: &CdcOffsetParseFunc,
363        upstream_chunk_buffer: &mut Vec<StreamChunk>,
364        current_pk_pos: Option<&OwnedRow>,
365        pk_compare: PkCompareInfo<'_>,
366        last_binlog_offset: &Option<CdcOffset>,
367        output_indices: &[usize],
368    ) -> StreamExecutorResult<(Vec<StreamChunk>, u64, Option<CdcOffset>)> {
369        let Some(current_pos) = current_pk_pos else {
370            return Ok((vec![], 0, None));
371        };
372
373        let buffered_chunks = std::mem::take(upstream_chunk_buffer);
374        let mut emitted_chunks = Vec::with_capacity(buffered_chunks.len());
375        let mut upstream_processed_row_count = 0;
376        let mut consumed_binlog_offset = None;
377        let mut retained_chunks = Vec::with_capacity(buffered_chunks.len());
378        let mut can_advance_consumed_offset = true;
379
380        for chunk in buffered_chunks {
381            let mut emitted_vis = BitmapBuilder::zeroed(chunk.capacity());
382            let mut retained_vis = BitmapBuilder::zeroed(chunk.capacity());
383            for idx in 0..chunk.capacity() {
384                let (op, row, visible) = chunk.row_at(idx);
385                if !visible {
386                    // Buffered chunks may carry sparse visibility from previous filtering rounds.
387                    continue;
388                }
389                let event_offset = (*offset_parse_func)(
390                    row.iter()
391                        .last()
392                        .flatten()
393                        .expect("cdc offset must exist")
394                        .into_utf8(),
395                )?;
396                let in_binlog_range = last_binlog_offset
397                    .as_ref()
398                    .is_none_or(|binlog_low| *binlog_low <= event_offset);
399
400                let row_pk = row.project(pk_compare.indices);
401                let reached_current_pos = cmp_pk_unsigned_aware(
402                    row_pk.iter(),
403                    current_pos.iter(),
404                    pk_compare.order,
405                    pk_compare.needs_unsigned_i64_compare,
406                )
407                .is_le();
408                if !in_binlog_range {
409                    continue;
410                }
411                let should_emit = reached_current_pos;
412
413                match op {
414                    Op::Insert | Op::Delete => {
415                        if should_emit {
416                            emitted_vis.set(idx, true);
417                            if can_advance_consumed_offset {
418                                consumed_binlog_offset = Some(event_offset);
419                            }
420                            upstream_processed_row_count += 1;
421                        } else {
422                            retained_vis.set(idx, true);
423                            can_advance_consumed_offset = false;
424                        }
425                    }
426                    Op::UpdateDelete | Op::UpdateInsert => {
427                        unreachable!("CDC buffered chunks should not contain update pairs")
428                    }
429                }
430            }
431
432            let emitted_vis = emitted_vis.finish();
433            if emitted_vis.count_ones() > 0 {
434                emitted_chunks.push(mapping_chunk(
435                    chunk.clone_with_vis(emitted_vis),
436                    output_indices,
437                ));
438            }
439
440            let retained_vis = retained_vis.finish();
441            if retained_vis.count_ones() > 0 {
442                retained_chunks.push(chunk.clone_with_vis(retained_vis));
443            }
444        }
445
446        *upstream_chunk_buffer = retained_chunks;
447
448        Ok((
449            emitted_chunks,
450            upstream_processed_row_count,
451            consumed_binlog_offset,
452        ))
453    }
454
455    /// Returned chunk has rows safe to send downstream marked as visible, unsafe rows marked invisible.
456    ///
457    /// Rows are safe to send downstream if in binlog range and `PK <= current_pk_pos`.
458    fn filter_recovery_chunk(
459        offset_parse_func: &CdcOffsetParseFunc,
460        chunk: StreamChunk,
461        current_pos: &OwnedRow,
462        pk_compare: PkCompareInfo<'_>,
463        last_binlog_offset: &Option<CdcOffset>,
464        output_indices: &[usize],
465    ) -> StreamExecutorResult<(Option<StreamChunk>, Option<CdcOffset>)> {
466        let chunk_offset = get_cdc_chunk_last_offset(offset_parse_func, &chunk)?;
467        let chunk = mapping_chunk(
468            mark_cdc_chunk(
469                offset_parse_func,
470                chunk,
471                current_pos,
472                pk_compare.indices,
473                pk_compare.order,
474                pk_compare.needs_unsigned_i64_compare,
475                last_binlog_offset.clone(),
476            )?,
477            output_indices,
478        );
479        let chunk = (chunk.cardinality() > 0).then_some(chunk);
480        let consumed_offset = chunk_offset.filter(|chunk_offset| {
481            last_binlog_offset
482                .as_ref()
483                .is_none_or(|last| *last < *chunk_offset)
484        });
485        Ok((chunk, consumed_offset))
486    }
487
488    #[try_stream(ok = Message, error = StreamExecutorError)]
489    async fn execute_inner(mut self) {
490        // The indices to primary key columns
491        let pk_indices = self.external_table.pk_indices().to_vec();
492        let pk_order = self.external_table.pk_order_types().to_vec();
493        let pk_needs_unsigned_i64_compare =
494            self.external_table.pk_comparisons().map(|comparisons| {
495                comparisons
496                    .iter()
497                    .map(|comparison| *comparison == CdcKeyComparison::UnsignedInt64)
498                    .collect_vec()
499            });
500
501        let table_id = self.external_table.table_id();
502        let upstream_table_name = self.external_table.qualified_table_name();
503        let schema_table_name = self.external_table.schema_table_name().clone();
504        let external_database_name = self.external_table.database_name().to_owned();
505
506        let additional_columns = self
507            .output_columns
508            .iter()
509            .filter(|col| col.additional_column.column_type.is_some())
510            .cloned()
511            .collect_vec();
512
513        let mut upstream = self.upstream.execute();
514
515        // Current position of the upstream_table storage primary key.
516        // `None` means it starts from the beginning.
517        let mut current_pk_pos: Option<OwnedRow>;
518
519        // Poll the upstream to get the first barrier.
520        let first_barrier = expect_first_barrier(&mut upstream).await?;
521
522        let mut is_snapshot_paused = first_barrier.is_pause_on_startup();
523        let first_barrier_epoch = first_barrier.epoch;
524        // The first barrier message should be propagated.
525        yield Message::Barrier(first_barrier);
526
527        let mut state_impl = self.state_impl;
528
529        state_impl.init_epoch(first_barrier_epoch).await?;
530
531        // restore backfill state
532        let state = state_impl.restore_state().await?;
533        current_pk_pos = state.current_pk_pos.clone();
534
535        let need_backfill = !self.options.disable_backfill && !state.is_finished;
536
537        // Keep track of rows from the snapshot.
538        let mut total_snapshot_row_count = state.row_count as u64;
539
540        // Only postgres-cdc connector may trigger TOAST.
541        let handle_postgres_toast_columns: bool =
542            self.external_table.table_type() == &ExternalCdcTableType::Postgres;
543        // Make sure to use mapping_message after transform_upstream.
544        let mut upstream = transform_upstream(
545            upstream,
546            self.output_columns.clone(),
547            self.properties.clone(),
548            handle_postgres_toast_columns,
549        )
550        .boxed()
551        .peekable();
552        let mut last_binlog_offset = state.last_cdc_offset.clone();
553
554        // CDC Backfill Algorithm:
555        //
556        // When the first barrier comes from upstream:
557        //  - read the current binlog offset as `binlog_low`
558        //  - start a snapshot read upon upstream table and iterate over the snapshot read stream
559        //  - buffer the changelog event from upstream
560        //
561        // When a new barrier comes from upstream:
562        //  - read the current binlog offset as `binlog_high`
563        //  - for each row of the upstream change log, forward it to downstream if it in the range
564        //    of [binlog_low, binlog_high] and its pk <= `current_pos`, otherwise keep buffering it
565        //    until `current_pos` catches up
566        //  - reconstruct the whole backfill stream with upstream changelog and a new table snapshot
567        //
568        // When a chunk comes from snapshot, we forward it to the downstream and raise
569        // `current_pos`.
570        // When we reach the end of the snapshot read stream, it means backfill has been
571        // finished.
572        //
573        // Once the backfill loop ends, we forward the upstream directly to the downstream.
574        if need_backfill {
575            // After init the state table and forward the initial barrier to downstream,
576            // we now try to create the table reader with retry.
577            //
578            // A fresh backfill can ignore CDC events here because its snapshot starts from the
579            // beginning. Recovery preserves events for the already-scanned prefix while reader
580            // creation is retried. Legacy MySQL graphs with Int64 PK columns must first recover
581            // comparison metadata from the reader so events use unsigned-aware ordering if needed.
582            let mut offset_parse_func = self.external_table.table_type().get_cdc_offset_parser()?;
583            let mut future = Box::pin(create_table_reader_with_retry(
584                self.external_table.clone(),
585                self.actor_ctx.id,
586                self.actor_ctx.fragment_id,
587            ));
588            let table_reader = if can_poll_upstream_while_creating_reader(
589                current_pk_pos.as_ref(),
590                pk_needs_unsigned_i64_compare.is_some(),
591            ) {
592                let mut handler = NoSnapshotUpstreamHandler {
593                    actor_ctx: &self.actor_ctx,
594                    metrics: &self.metrics,
595                    table_id,
596                    upstream_table_name: &upstream_table_name,
597                    offset_parse_func: &mut offset_parse_func,
598                    output_indices: &self.output_indices,
599                    pk_indices: &pk_indices,
600                    pk_order: &pk_order,
601                    cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
602                        pk_pos,
603                        pk_needs_unsigned_i64_compare: pk_needs_unsigned_i64_compare
604                            .as_deref()
605                            .expect("checked by can_poll_upstream_while_creating_reader"),
606                    }),
607                    total_snapshot_row_count,
608                    state_impl: &mut state_impl,
609                    last_binlog_offset: &mut last_binlog_offset,
610                    is_snapshot_paused: &mut is_snapshot_paused,
611                    rate_limit_rps: &mut self.rate_limit_rps,
612                };
613                loop {
614                    match handler.wait_for_reader(&mut upstream, &mut future).await? {
615                        Either::Left(NoSnapshotUpstreamOutcome::Continue(msg)) => {
616                            if let Some(msg) = msg {
617                                yield msg;
618                            }
619                        }
620                        Either::Left(NoSnapshotUpstreamOutcome::Stop(barrier)) => {
621                            yield Message::Barrier(barrier);
622
623                            return Ok(());
624                        }
625                        Either::Right(table_reader) => break table_reader,
626                    }
627                }
628            } else {
629                tracing::warn!(
630                    %table_id,
631                    upstream_table_name,
632                    "waiting for the CDC table reader to recover legacy MySQL primary-key comparison metadata; checkpoint progress is paused"
633                );
634                future.as_mut().await
635            };
636            let pk_names = self.external_table.pk_names();
637            let pk_needs_unsigned_i64_compare = match pk_needs_unsigned_i64_compare {
638                Some(comparisons) => comparisons,
639                None => {
640                    let comparisons = table_reader.pk_column_comparisons(&pk_names)?;
641                    assert_eq!(comparisons.len(), pk_indices.len());
642                    comparisons
643                        .into_iter()
644                        .map(|comparison| comparison == CdcKeyComparison::UnsignedInt64)
645                        .collect_vec()
646                }
647            };
648            tracing::info!(
649                %table_id,
650                upstream_table_name,
651                "table reader created successfully"
652            );
653
654            let mut upstream_table_reader =
655                UpstreamTableReader::new(self.external_table.clone(), table_reader);
656
657            if last_binlog_offset.is_none() {
658                // Limit concurrent CDC connections globally to 10 using a semaphore.
659                static CDC_CONN_SEMAPHORE: tokio::sync::Semaphore =
660                    tokio::sync::Semaphore::const_new(10);
661
662                let _permit = CDC_CONN_SEMAPHORE.acquire().await.unwrap();
663                last_binlog_offset = upstream_table_reader.current_cdc_offset().await?;
664            }
665
666            tracing::info!(
667                %table_id,
668                upstream_table_name,
669                initial_binlog_offset = ?last_binlog_offset,
670                ?current_pk_pos,
671                is_finished = state.is_finished,
672                is_snapshot_paused,
673                snapshot_row_count = total_snapshot_row_count,
674                rate_limit = self.rate_limit_rps,
675                disable_backfill = self.options.disable_backfill,
676                snapshot_barrier_interval = self.options.snapshot_barrier_interval,
677                snapshot_batch_size = self.options.snapshot_batch_size,
678                "start cdc backfill",
679            );
680
681            // drive the upstream changelog first to ensure we can receive timely changelog event,
682            // otherwise the upstream changelog may be blocked by the snapshot read stream
683            let _ = Pin::new(&mut upstream).peek().await;
684
685            // wait for a barrier to make sure the backfill starts after upstream source
686            let mut handler = NoSnapshotUpstreamHandler {
687                actor_ctx: &self.actor_ctx,
688                metrics: &self.metrics,
689                table_id,
690                upstream_table_name: &upstream_table_name,
691                offset_parse_func: &mut offset_parse_func,
692                output_indices: &self.output_indices,
693                pk_indices: &pk_indices,
694                pk_order: &pk_order,
695                cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
696                    pk_pos,
697                    pk_needs_unsigned_i64_compare: &pk_needs_unsigned_i64_compare,
698                }),
699                total_snapshot_row_count,
700                state_impl: &mut state_impl,
701                last_binlog_offset: &mut last_binlog_offset,
702                is_snapshot_paused: &mut is_snapshot_paused,
703                rate_limit_rps: &mut self.rate_limit_rps,
704            };
705            #[for_await]
706            for msg in upstream.by_ref() {
707                match msg? {
708                    Message::Chunk(chunk) if handler.cursor.is_none() => {
709                        // Without a snapshot cursor, the snapshot covers these changes, but the
710                        // binlog offset still advances past them.
711                        if chunk.has_visible_rows()
712                            && let Some(chunk_offset) =
713                                get_cdc_chunk_last_offset(handler.offset_parse_func, &chunk)?
714                            && handler
715                                .last_binlog_offset
716                                .as_ref()
717                                .is_none_or(|last| *last < chunk_offset)
718                        {
719                            *handler.last_binlog_offset = Some(chunk_offset);
720                        }
721                    }
722                    msg => match handler.handle_upstream(msg).await? {
723                        NoSnapshotUpstreamOutcome::Continue(Some(msg)) => {
724                            let is_barrier = matches!(msg, Message::Barrier(_));
725
726                            yield msg;
727
728                            if is_barrier {
729                                break;
730                            }
731                        }
732                        NoSnapshotUpstreamOutcome::Continue(None) => {}
733                        NoSnapshotUpstreamOutcome::Stop(barrier) => {
734                            yield Message::Barrier(barrier);
735
736                            return Ok(());
737                        }
738                    },
739                }
740            }
741
742            tracing::info!(%table_id,
743                upstream_table_name,
744                initial_binlog_offset = ?last_binlog_offset,
745                ?current_pk_pos,
746                is_snapshot_paused,
747                "start cdc backfill loop");
748
749            // the buffer will be drained when a barrier comes
750            let mut upstream_chunk_buffer: Vec<StreamChunk> = vec![];
751            let mut needs_reader_rebuild = false;
752
753            'backfill_loop: loop {
754                if needs_reader_rebuild {
755                    needs_reader_rebuild = false;
756                    tracing::info!(
757                        %table_id,
758                        upstream_table_name,
759                        "rebuilding CDC table reader after snapshot read failure"
760                    );
761
762                    if let Err(error) = upstream_table_reader.disconnect().await {
763                        tracing::warn!(
764                            error = %error.as_report(),
765                            %table_id,
766                            upstream_table_name,
767                            "failed to disconnect CDC snapshot reader; continuing with rebuild"
768                        );
769                    }
770
771                    let mut future = Box::pin(create_table_reader_with_retry(
772                        self.external_table.clone(),
773                        self.actor_ctx.id,
774                        self.actor_ctx.fragment_id,
775                    ));
776
777                    let mut handler = NoSnapshotUpstreamHandler {
778                        actor_ctx: &self.actor_ctx,
779                        metrics: &self.metrics,
780                        table_id,
781                        upstream_table_name: &upstream_table_name,
782                        offset_parse_func: &mut offset_parse_func,
783                        output_indices: &self.output_indices,
784                        pk_indices: &pk_indices,
785                        pk_order: &pk_order,
786                        cursor: current_pk_pos.as_ref().map(|pk_pos| SnapshotCursor {
787                            pk_pos,
788                            pk_needs_unsigned_i64_compare: &pk_needs_unsigned_i64_compare,
789                        }),
790                        total_snapshot_row_count,
791                        state_impl: &mut state_impl,
792                        last_binlog_offset: &mut last_binlog_offset,
793                        is_snapshot_paused: &mut is_snapshot_paused,
794                        rate_limit_rps: &mut self.rate_limit_rps,
795                    };
796                    let table_reader = loop {
797                        match handler.wait_for_reader(&mut upstream, &mut future).await? {
798                            Either::Left(NoSnapshotUpstreamOutcome::Continue(msg)) => {
799                                if let Some(msg) = msg {
800                                    yield msg;
801                                }
802                            }
803                            Either::Left(NoSnapshotUpstreamOutcome::Stop(barrier)) => {
804                                yield Message::Barrier(barrier);
805
806                                return Ok(());
807                            }
808                            Either::Right(table_reader) => break table_reader,
809                        }
810                    };
811
812                    upstream_table_reader =
813                        UpstreamTableReader::new(self.external_table.clone(), table_reader);
814                    tracing::info!(
815                        %table_id,
816                        upstream_table_name,
817                        "CDC table reader rebuilt successfully"
818                    );
819                }
820
821                let mut should_bypass_snapshot_stream_patch =
822                    self.rate_limit_rps.is_some_and(|val| val == 0);
823                let left_upstream = upstream.by_ref().map(Either::Left);
824
825                let mut snapshot_read_row_cnt: usize = 0;
826                let read_args = SnapshotReadArgs::new(
827                    current_pk_pos.clone(),
828                    self.rate_limit_rps,
829                    pk_indices.clone(),
830                    additional_columns.clone(),
831                    schema_table_name.clone(),
832                    external_database_name.clone(),
833                );
834                let right_snapshot = pin!(
835                    upstream_table_reader
836                        .snapshot_read_full_table(read_args, self.options.snapshot_batch_size)
837                        .map(Either::Right)
838                );
839                let (right_snapshot, snapshot_valve) = pausable(right_snapshot);
840                if is_snapshot_paused {
841                    snapshot_valve.pause();
842                }
843
844                // Prefer to select upstream, so we can stop snapshot stream when barrier comes.
845                let mut backfill_stream =
846                    select_with_strategy(left_upstream, right_snapshot, |_: &mut ()| {
847                        stream::PollNext::Left
848                    });
849
850                let mut cur_barrier_snapshot_processed_rows: u64 = 0;
851                let mut cur_barrier_upstream_processed_rows: u64 = 0;
852                let mut barrier_count: u32 = 0;
853                let mut pending_barrier = None;
854
855                #[for_await]
856                for either in &mut backfill_stream {
857                    match either {
858                        // Upstream
859                        Either::Left(msg) => {
860                            match msg? {
861                                Message::Barrier(barrier) => {
862                                    // increase the barrier count and check whether need to start a new snapshot
863                                    barrier_count += 1;
864                                    let mut needs_rebuild_snapshot =
865                                        barrier_count == self.options.snapshot_barrier_interval;
866
867                                    if let Some(mutation) = barrier.mutation.as_deref() {
868                                        match mutation {
869                                            Mutation::Pause => {
870                                                is_snapshot_paused = true;
871                                                snapshot_valve.pause();
872                                            }
873                                            Mutation::Resume => {
874                                                is_snapshot_paused = false;
875                                                snapshot_valve.resume();
876                                            }
877                                            Mutation::Throttle(_) => {
878                                                if let Some(entry) = mutation
879                                                    .backfill_throttle_config(
880                                                        self.actor_ctx.fragment_id,
881                                                    )
882                                                    && entry.rate_limit != self.rate_limit_rps
883                                                {
884                                                    // If self.rate_limit_rps is 0, the snapshot stream does not establish a snapshot_read.
885                                                    // Consequently, the later snapshot stream patch is bypassed.
886                                                    should_bypass_snapshot_stream_patch = self
887                                                        .rate_limit_rps
888                                                        .is_some_and(|val| val == 0);
889                                                    self.rate_limit_rps = entry.rate_limit;
890                                                    needs_rebuild_snapshot = true;
891                                                }
892                                            }
893                                            mutation if mutation.is_stop(self.actor_ctx.id) => {
894                                                // the actor has been dropped, exit the backfill loop
895                                                tracing::info!(
896                                                    %table_id,
897                                                    upstream_table_name,
898                                                    "CdcBackfill has been dropped due to config change"
899                                                );
900                                                state_impl.commit_state(barrier.epoch).await?;
901                                                yield Message::Barrier(barrier);
902                                                break 'backfill_loop;
903                                            }
904                                            _ => (),
905                                        }
906                                    }
907
908                                    // when processing a barrier, check whether can start a new snapshot
909                                    // if the number of barriers reaches the snapshot interval
910                                    if needs_rebuild_snapshot || needs_reader_rebuild {
911                                        // staging the barrier
912                                        pending_barrier = Some(barrier);
913                                        tracing::debug!(
914                                            %table_id,
915                                            ?current_pk_pos,
916                                            ?snapshot_read_row_cnt,
917                                            "Prepare to start a new snapshot"
918                                        );
919                                        // Break the loop for consuming snapshot and prepare to start a new snapshot
920                                        break;
921                                    } else {
922                                        // Drain the in-memory buffer to ensure no data is lost during the recovery process.
923                                        let (
924                                            emitted_upstream_chunks,
925                                            consumed_upstream_row_count,
926                                            consumed_binlog_offset,
927                                        ) = Self::consume_upstream_chunk_buffer(
928                                            &offset_parse_func,
929                                            &mut upstream_chunk_buffer,
930                                            current_pk_pos.as_ref(),
931                                            PkCompareInfo {
932                                                indices: &pk_indices,
933                                                order: &pk_order,
934                                                needs_unsigned_i64_compare:
935                                                    &pk_needs_unsigned_i64_compare,
936                                            },
937                                            &last_binlog_offset,
938                                            &self.output_indices,
939                                        )?;
940                                        cur_barrier_upstream_processed_rows +=
941                                            consumed_upstream_row_count;
942                                        if let Some(consumed_binlog_offset) = consumed_binlog_offset
943                                        {
944                                            last_binlog_offset = Some(consumed_binlog_offset);
945                                        }
946                                        for chunk in emitted_upstream_chunks {
947                                            yield Message::Chunk(chunk);
948                                        }
949
950                                        Self::report_metrics(
951                                            &self.metrics,
952                                            cur_barrier_snapshot_processed_rows,
953                                            cur_barrier_upstream_processed_rows,
954                                        );
955
956                                        // update and persist current backfill progress
957                                        state_impl
958                                            .mutate_and_commit_state(
959                                                current_pk_pos.clone(),
960                                                last_binlog_offset.clone(),
961                                                total_snapshot_row_count,
962                                                false,
963                                                barrier.epoch,
964                                            )
965                                            .await?;
966
967                                        // emit barrier and continue consume the backfill stream
968                                        yield Message::Barrier(barrier);
969                                    }
970                                }
971                                Message::Chunk(chunk) => {
972                                    // skip empty upstream chunk
973                                    if chunk.cardinality() == 0 {
974                                        continue;
975                                    }
976
977                                    let chunk_binlog_offset =
978                                        get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
979
980                                    tracing::trace!(
981                                        "recv changelog chunk: chunk_offset {:?}, capactiy {}",
982                                        chunk_binlog_offset,
983                                        chunk.capacity()
984                                    );
985
986                                    // Since we don't need changelog before the
987                                    // `last_binlog_offset`, skip the chunk that *only* contains
988                                    // events before `last_binlog_offset`.
989                                    if let (Some(last_binlog_offset), Some(chunk_offset)) =
990                                        (last_binlog_offset.as_ref(), chunk_binlog_offset)
991                                        && chunk_offset < *last_binlog_offset
992                                    {
993                                        tracing::trace!(
994                                            "skip changelog chunk: chunk_offset {:?}, capacity {}",
995                                            chunk_offset,
996                                            chunk.capacity()
997                                        );
998                                        continue;
999                                    }
1000
1001                                    // Buffer the upstream chunk.
1002                                    upstream_chunk_buffer.push(chunk.compact_vis());
1003                                }
1004                                Message::Watermark(_) => {
1005                                    // Ignore watermark during backfill.
1006                                }
1007                            }
1008                        }
1009                        // Snapshot read
1010                        Either::Right(msg) => {
1011                            match msg {
1012                                Ok(None) => {
1013                                    tracing::info!(
1014                                        %table_id,
1015                                        ?last_binlog_offset,
1016                                        ?current_pk_pos,
1017                                        "snapshot read stream ends"
1018                                    );
1019                                    // If the snapshot read stream ends, it means all historical
1020                                    // data has been loaded.
1021                                    // We should not mark the chunk anymore,
1022                                    // otherwise, we will ignore some rows in the buffer.
1023                                    for chunk in upstream_chunk_buffer.drain(..) {
1024                                        yield Message::Chunk(mapping_chunk(
1025                                            chunk,
1026                                            &self.output_indices,
1027                                        ));
1028                                    }
1029
1030                                    // backfill has finished, exit the backfill loop and persist the state when we recv a barrier
1031                                    break 'backfill_loop;
1032                                }
1033                                Ok(Some(chunk)) => {
1034                                    // Raise the current position.
1035                                    // As snapshot read streams are ordered by pk, so we can
1036                                    // just use the last row to update `current_pos`.
1037                                    current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
1038
1039                                    tracing::trace!(
1040                                        "got a snapshot chunk: len {}, current_pk_pos {:?}",
1041                                        chunk.cardinality(),
1042                                        current_pk_pos
1043                                    );
1044                                    let chunk_cardinality = chunk.cardinality() as u64;
1045                                    cur_barrier_snapshot_processed_rows += chunk_cardinality;
1046                                    total_snapshot_row_count += chunk_cardinality;
1047                                    yield Message::Chunk(mapping_chunk(
1048                                        chunk,
1049                                        &self.output_indices,
1050                                    ));
1051                                }
1052                                Err(error) => {
1053                                    needs_reader_rebuild = true;
1054                                    tracing::warn!(
1055                                        error = %error.as_report(),
1056                                        %table_id,
1057                                        upstream_table_name,
1058                                        "failed to read CDC snapshot; rebuilding reader after a barrier"
1059                                    );
1060                                }
1061                            }
1062                        }
1063                    }
1064                }
1065
1066                assert!(pending_barrier.is_some(), "pending_barrier must exist");
1067                let pending_barrier = pending_barrier.unwrap();
1068
1069                // The snapshot stream patch:
1070                // Here we have to ensure the snapshot stream is consumed at least once,
1071                // since the barrier event can kick in anytime.
1072                // Otherwise, the result set of the new snapshot stream may become empty.
1073                // It maybe a cancellation bug of the mysql driver.
1074                let (_, mut snapshot_stream) = backfill_stream.into_inner();
1075                // Resume the snapshot stream so that the snapshot stream patch won't block.
1076                if !should_bypass_snapshot_stream_patch && is_snapshot_paused {
1077                    snapshot_valve.resume();
1078                }
1079                if !should_bypass_snapshot_stream_patch
1080                    && let Some(msg) = snapshot_stream
1081                        .next()
1082                        .instrument_await("consume_snapshot_stream_once")
1083                        .await
1084                {
1085                    let Either::Right(msg) = msg else {
1086                        bail!("BUG: snapshot_read contains upstream messages");
1087                    };
1088                    match msg {
1089                        Ok(None) => {
1090                            tracing::info!(
1091                                %table_id,
1092                                ?last_binlog_offset,
1093                                ?current_pk_pos,
1094                                "snapshot read stream ends in the force emit branch"
1095                            );
1096                            // End of the snapshot read stream.
1097                            // Consume the buffered upstream chunk without filtering by `binlog_low`.
1098                            for chunk in upstream_chunk_buffer.drain(..) {
1099                                yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
1100                            }
1101
1102                            // mark backfill has finished
1103                            state_impl
1104                                .mutate_and_commit_state(
1105                                    current_pk_pos.clone(),
1106                                    last_binlog_offset.clone(),
1107                                    total_snapshot_row_count,
1108                                    true,
1109                                    pending_barrier.epoch,
1110                                )
1111                                .await?;
1112
1113                            yield Message::Barrier(pending_barrier);
1114                            // end of backfill loop, since backfill has finished
1115                            break 'backfill_loop;
1116                        }
1117                        Ok(Some(_)) if is_snapshot_paused => {
1118                            // Since the snapshot stream is paused, drop the chunk.
1119                        }
1120                        Ok(Some(chunk)) => {
1121                            // Raise the current pk position.
1122                            current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
1123
1124                            let row_count = chunk.cardinality() as u64;
1125                            cur_barrier_snapshot_processed_rows += row_count;
1126                            total_snapshot_row_count += row_count;
1127                            snapshot_read_row_cnt += row_count as usize;
1128
1129                            tracing::debug!(
1130                                %table_id,
1131                                ?current_pk_pos,
1132                                ?snapshot_read_row_cnt,
1133                                "force emit a snapshot chunk"
1134                            );
1135                            yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
1136                        }
1137                        Err(error) => {
1138                            needs_reader_rebuild = true;
1139                            tracing::warn!(
1140                                error = %error.as_report(),
1141                                %table_id,
1142                                upstream_table_name,
1143                                "failed to read CDC snapshot; rebuilding reader after a barrier"
1144                            );
1145                        }
1146                    }
1147                }
1148
1149                // If the number of barriers reaches the snapshot interval,
1150                // consume the buffered upstream chunks.
1151                let mut consumed_binlog_offset = None;
1152
1153                if let Some(current_pos) = &current_pk_pos {
1154                    for chunk in upstream_chunk_buffer.drain(..) {
1155                        cur_barrier_upstream_processed_rows += chunk.cardinality() as u64;
1156
1157                        // record the consumed binlog offset that will be
1158                        // persisted later
1159                        consumed_binlog_offset =
1160                            get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
1161
1162                        yield Message::Chunk(mapping_chunk(
1163                            mark_cdc_chunk(
1164                                &offset_parse_func,
1165                                chunk,
1166                                current_pos,
1167                                &pk_indices,
1168                                &pk_order,
1169                                &pk_needs_unsigned_i64_compare,
1170                                last_binlog_offset.clone(),
1171                            )?,
1172                            &self.output_indices,
1173                        ));
1174                    }
1175                } else {
1176                    // If no current_pos, means we did not process any snapshot yet.
1177                    // we can just ignore the upstream buffer chunk in that case.
1178                    upstream_chunk_buffer.clear();
1179                }
1180
1181                // Update last seen binlog offset
1182                if consumed_binlog_offset.is_some() {
1183                    last_binlog_offset.clone_from(&consumed_binlog_offset);
1184                }
1185
1186                Self::report_metrics(
1187                    &self.metrics,
1188                    cur_barrier_snapshot_processed_rows,
1189                    cur_barrier_upstream_processed_rows,
1190                );
1191
1192                // update and persist current backfill progress
1193                state_impl
1194                    .mutate_and_commit_state(
1195                        current_pk_pos.clone(),
1196                        last_binlog_offset.clone(),
1197                        total_snapshot_row_count,
1198                        false,
1199                        pending_barrier.epoch,
1200                    )
1201                    .await?;
1202
1203                yield Message::Barrier(pending_barrier);
1204            }
1205            upstream_table_reader.disconnect().await?;
1206        } else if self.options.disable_backfill {
1207            // If backfill is disabled, we just mark the backfill as finished
1208            tracing::info!(
1209                %table_id,
1210                upstream_table_name,
1211                "CdcBackfill has been disabled"
1212            );
1213            state_impl
1214                .mutate_state(
1215                    current_pk_pos.clone(),
1216                    last_binlog_offset.clone(),
1217                    total_snapshot_row_count,
1218                    true,
1219                )
1220                .await?;
1221        }
1222
1223        tracing::info!(
1224            %table_id,
1225            upstream_table_name,
1226            "CdcBackfill has already finished and will forward messages directly to the downstream"
1227        );
1228
1229        // Wait for first barrier to come after backfill is finished.
1230        // So we can update our progress + persist the status.
1231        while let Some(Ok(msg)) = upstream.next().await {
1232            if let Some(msg) = mapping_message(msg, &self.output_indices) {
1233                // If not finished then we need to update state, otherwise no need.
1234                if let Message::Barrier(barrier) = &msg {
1235                    // finalized the backfill state
1236                    state_impl
1237                        .mutate_and_commit_state(
1238                            current_pk_pos.clone(),
1239                            last_binlog_offset.clone(),
1240                            total_snapshot_row_count,
1241                            true,
1242                            barrier.epoch,
1243                        )
1244                        .await?;
1245
1246                    // mark progress as finished
1247                    if let Some(progress) = self.progress.as_mut() {
1248                        progress.finish(barrier.epoch, total_snapshot_row_count);
1249                    }
1250                    yield msg;
1251                    // break after the state have been saved
1252                    break;
1253                }
1254                yield msg;
1255            }
1256        }
1257
1258        // After backfill progress finished
1259        // we can forward messages directly to the downstream,
1260        // as backfill is finished.
1261        #[for_await]
1262        for msg in upstream {
1263            // upstream offsets will be removed from the message before forwarding to
1264            // downstream
1265            if let Some(msg) = mapping_message(msg?, &self.output_indices) {
1266                if let Message::Barrier(barrier) = &msg {
1267                    // commit state just to bump the epoch of state table
1268                    state_impl.commit_state(barrier.epoch).await?;
1269                }
1270                yield msg;
1271            }
1272        }
1273    }
1274}
1275
1276#[try_stream(ok = Message, error = StreamExecutorError)]
1277pub async fn transform_upstream(
1278    upstream: BoxedMessageStream,
1279    output_columns: Vec<ColumnDesc>,
1280    properties: BTreeMap<String, String>,
1281    handle_postgres_toast_columns: bool,
1282) {
1283    let (timestamp_handling, timestamptz_handling, time_handling, bigint_unsigned_handling) =
1284        get_cdc_json_parse_handling_from_properties(&properties);
1285    let props = SpecificParserConfig {
1286        encoding_config: EncodingProperties::Json(JsonProperties {
1287            use_schema_registry: false,
1288            timestamp_handling,
1289            timestamptz_handling,
1290            time_handling,
1291            bigint_unsigned_handling,
1292            handle_postgres_toast_columns,
1293        }),
1294        // the cdc message is generated internally so the key must exist.
1295        protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()),
1296    };
1297
1298    // convert to source column desc to feed into parser
1299    let columns_with_meta = output_columns
1300        .iter()
1301        .map(SourceColumnDesc::from)
1302        .collect_vec();
1303    let mut parser = DebeziumParser::new(
1304        props,
1305        columns_with_meta.clone(),
1306        Arc::new(SourceContext::dummy()),
1307        properties.cdc_source_type().debezium_numeric_handling(),
1308    )
1309    .await
1310    .map_err(StreamExecutorError::connector_error)?;
1311
1312    pin_mut!(upstream);
1313    #[for_await]
1314    for msg in upstream {
1315        let mut msg = msg?;
1316        if let Message::Chunk(chunk) = &mut msg {
1317            let parsed_chunk = parse_debezium_chunk(&mut parser, chunk).await?;
1318            let _ = std::mem::replace(chunk, parsed_chunk);
1319        }
1320        yield msg;
1321    }
1322}
1323
1324async fn parse_debezium_chunk(
1325    parser: &mut DebeziumParser,
1326    chunk: &StreamChunk,
1327) -> StreamExecutorResult<StreamChunk> {
1328    // here we transform the input chunk in `(payload varchar, _rw_offset varchar, _rw_table_name varchar)` schema
1329    // to chunk with downstream table schema `info.schema` of MergeNode contains the schema of the
1330    // table job with `_rw_offset` in the end
1331    // see `gen_create_table_plan_for_cdc_source` for details
1332
1333    // use `SourceStreamChunkBuilder` for convenience
1334    let mut builder = SourceStreamChunkBuilder::new(
1335        parser.columns().to_vec(),
1336        SourceCtrlOpts {
1337            chunk_size: chunk.capacity(),
1338            split_txn: false,
1339        },
1340    );
1341
1342    // The schema of input chunk `(payload varchar, _rw_offset varchar, _rw_table_name varchar, _row_id)`
1343    // We should use the debezium parser to parse the first column,
1344    // then chain the parsed row with `_rw_offset` row to get a new row.
1345    let payloads = chunk.data_chunk().project(&[0]);
1346    let offsets = chunk.data_chunk().project(&[1]).compact_vis();
1347
1348    // TODO: preserve the transaction semantics
1349    for payload in payloads.rows() {
1350        let ScalarRefImpl::Jsonb(jsonb_ref) = payload.datum_at(0).expect("payload must exist")
1351        else {
1352            panic!("payload must be jsonb");
1353        };
1354
1355        parser
1356            .parse_inner(
1357                None,
1358                Some(jsonb_ref.to_string().as_bytes().to_vec()),
1359                builder.row_writer(),
1360            )
1361            .await
1362            .unwrap();
1363    }
1364    builder.finish_current_chunk();
1365
1366    let parsed_chunk = {
1367        let mut iter = builder.consume_ready_chunks();
1368        assert_eq!(1, iter.len());
1369        iter.next().unwrap()
1370    };
1371    assert_eq!(parsed_chunk.capacity(), chunk.capacity()); // each payload is expected to generate one row
1372    let (ops, mut columns, vis) = parsed_chunk.into_inner();
1373    // note that `vis` is not necessarily the same as the original chunk's visibilities
1374
1375    // concat the rows in the parsed chunk with the `_rw_offset` column
1376    columns.extend(offsets.into_parts().0);
1377
1378    Ok(StreamChunk::from_parts(
1379        ops,
1380        DataChunk::from_parts(columns.into(), vis),
1381    ))
1382}
1383
1384impl<S: StateStore> Execute for CdcBackfillExecutor<S> {
1385    fn execute(self: Box<Self>) -> BoxedMessageStream {
1386        self.execute_inner().boxed()
1387    }
1388}
1389
1390#[cfg(test)]
1391mod tests {
1392    use std::collections::BTreeMap;
1393    use std::str::FromStr;
1394    use std::time::Duration;
1395
1396    use either::Either;
1397    use futures::{StreamExt, pin_mut, stream};
1398    use risingwave_common::array::{Array, DataChunk, Op, StreamChunk};
1399    use risingwave_common::catalog::{
1400        CdcKeyComparison, ColumnDesc, ColumnId, Field, Schema, TableId,
1401    };
1402    use risingwave_common::row::{OwnedRow, Row};
1403    use risingwave_common::types::{DataType, Datum, JsonbVal, ScalarImpl};
1404    use risingwave_common::util::epoch::test_epoch;
1405    use risingwave_common::util::iter_util::ZipEqFast;
1406    use risingwave_common::util::sort_util::OrderType;
1407    use risingwave_connector::source::cdc::CdcScanOptions;
1408    use risingwave_connector::source::cdc::external::mock_external_table::MockExternalTableReader;
1409    use risingwave_connector::source::cdc::external::mysql::MysqlOffset;
1410    use risingwave_connector::source::cdc::external::{
1411        CdcOffset, ExternalCdcTableType, ExternalTableConfig, ExternalTableReaderImpl,
1412        SchemaTableName,
1413    };
1414    use risingwave_storage::memory::MemoryStateStore;
1415
1416    use super::{
1417        PkCompareInfo, build_reader_and_poll_upstream, can_poll_upstream_while_creating_reader,
1418    };
1419    use crate::common::table::test_utils::gen_pbtable;
1420    use crate::executor::backfill::cdc::cdc_backfill::transform_upstream;
1421    use crate::executor::backfill::cdc::state::CdcBackfillState;
1422    use crate::executor::monitor::StreamingMetrics;
1423    use crate::executor::prelude::StateTable;
1424    use crate::executor::test_utils::{MessageSender, MockSource};
1425    use crate::executor::{
1426        ActorContext, Barrier, CdcBackfillExecutor, ExternalStorageTable, Message,
1427    };
1428
1429    #[tokio::test]
1430    async fn test_transform_upstream_chunk() {
1431        let schema = Schema::new(vec![
1432            Field::unnamed(DataType::Jsonb),   // debezium json payload
1433            Field::unnamed(DataType::Varchar), // _rw_offset
1434            Field::unnamed(DataType::Varchar), // _rw_table_name
1435        ]);
1436        let stream_key = vec![1];
1437        let (mut tx, source) = MockSource::channel();
1438        let source = source.into_executor(schema.clone(), stream_key.clone());
1439        // let payload = r#"{"before": null,"after":{"O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" },"source":{"version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null},"op":"r","ts_ms":1695277757017,"transaction":null}"#.to_string();
1440        let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#;
1441
1442        let datums: Vec<Datum> = vec![
1443            Some(JsonbVal::from_str(payload).unwrap().into()),
1444            Some("file: 1.binlog, pos: 100".to_owned().into()),
1445            Some("mydb.orders".to_owned().into()),
1446        ];
1447
1448        let mut builders = schema.create_array_builders(8);
1449        for (builder, datum) in builders.iter_mut().zip_eq_fast(datums.iter()) {
1450            builder.append(datum.clone());
1451        }
1452        let columns = builders
1453            .into_iter()
1454            .map(|builder| builder.finish().into())
1455            .collect();
1456
1457        // one row chunk
1458        let chunk = StreamChunk::from_parts(vec![Op::Insert], DataChunk::new(columns, 1));
1459
1460        tx.push_chunk(chunk);
1461        let upstream = Box::new(source).execute();
1462
1463        // schema to the debezium parser
1464        let columns = vec![
1465            ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
1466            ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
1467            ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
1468            ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
1469            ColumnDesc::named("O_ORDERDATE", ColumnId::new(5), DataType::Date),
1470            ColumnDesc::named("commit_ts", ColumnId::new(6), DataType::Timestamptz),
1471        ];
1472
1473        let parsed_stream = transform_upstream(upstream, columns, BTreeMap::new(), false);
1474        pin_mut!(parsed_stream);
1475        let message = parsed_stream
1476            .next()
1477            .await
1478            .expect("transform stream should yield the input chunk")
1479            .expect("transforming the CDC chunk should succeed");
1480        let Message::Chunk(chunk) = message else {
1481            panic!("expected a transformed chunk");
1482        };
1483        assert_eq!(
1484            chunk
1485                .rows()
1486                .map(|(op, row)| (op, row.to_owned_row()))
1487                .collect::<Vec<_>>(),
1488            vec![(
1489                Op::Insert,
1490                OwnedRow::new(vec![
1491                    Some(ScalarImpl::Int64(5)),
1492                    Some(ScalarImpl::Int64(44485)),
1493                    Some(ScalarImpl::Utf8("F".into())),
1494                    Some(ScalarImpl::Decimal("144659.20".parse().unwrap())),
1495                    Some(ScalarImpl::Date("1994-07-30".parse().unwrap())),
1496                    None,
1497                    Some(ScalarImpl::Utf8("file: 1.binlog, pos: 100".into())),
1498                ]),
1499            )]
1500        );
1501    }
1502
1503    #[tokio::test]
1504    async fn test_finished_backfill_forwards_without_reader() {
1505        let actor_context = ActorContext::for_test(1);
1506        let table_id = TableId::new(1234);
1507        let external_storage_table = ExternalStorageTable::new(
1508            table_id,
1509            SchemaTableName {
1510                schema_name: "public".to_owned(),
1511                table_name: "missing_table".to_owned(),
1512            },
1513            "mydb".to_owned(),
1514            ExternalTableConfig::default(),
1515            ExternalCdcTableType::Unspecified,
1516            Schema::new(vec![Field::with_name(DataType::Int64, "O_ORDERKEY")]),
1517            vec![OrderType::ascending()],
1518            Some(vec![CdcKeyComparison::Native]),
1519            vec![0],
1520        );
1521        let schema = Schema::new(vec![
1522            Field::unnamed(DataType::Jsonb),   // debezium json payload
1523            Field::unnamed(DataType::Varchar), // _rw_offset
1524            Field::unnamed(DataType::Varchar), // _rw_table_name
1525        ]);
1526        let stream_key = vec![1];
1527        let (mut tx, source) = MockSource::channel();
1528        let source = source
1529            .stop_on_finish(false)
1530            .into_executor(schema.clone(), stream_key.clone());
1531        let output_indices = vec![1, 0, 4]; //reorder
1532        let output_columns = vec![
1533            ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
1534            ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
1535            ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
1536            ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
1537            ColumnDesc::named("O_DUMMY", ColumnId::new(5), DataType::Int64),
1538            ColumnDesc::named("commit_ts", ColumnId::new(6), DataType::Timestamptz),
1539        ];
1540        let store = MemoryStateStore::new();
1541        let mut persisted_state =
1542            CdcBackfillState::new(table_id, create_cdc_state_table(store.clone()).await, 5);
1543        persisted_state
1544            .init_epoch(Barrier::new_test_barrier(test_epoch(6)).epoch)
1545            .await
1546            .unwrap();
1547        persisted_state
1548            .mutate_and_commit_state(
1549                Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
1550                Some(CdcOffset::Mysql(MysqlOffset::new(
1551                    "1.binlog".to_owned(),
1552                    100,
1553                ))),
1554                1,
1555                true,
1556                Barrier::new_test_barrier(test_epoch(7)).epoch,
1557            )
1558            .await
1559            .unwrap();
1560        let state_table = create_cdc_state_table(store).await;
1561        let cdc = CdcBackfillExecutor::new(
1562            actor_context,
1563            external_storage_table,
1564            source,
1565            output_indices,
1566            output_columns,
1567            None,
1568            StreamingMetrics::unused().into(),
1569            state_table,
1570            None,
1571            CdcScanOptions::default(),
1572            BTreeMap::default(),
1573        );
1574        let s = cdc.execute_inner();
1575        pin_mut!(s);
1576
1577        // send first barrier
1578        tx.send_barrier(Barrier::new_test_barrier(test_epoch(8)));
1579        // send chunk
1580        {
1581            let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_DUMMY": 100 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#;
1582            let datums: Vec<Datum> = vec![
1583                Some(JsonbVal::from_str(payload).unwrap().into()),
1584                Some("file: 1.binlog, pos: 100".to_owned().into()),
1585                Some("mydb.orders".to_owned().into()),
1586            ];
1587            let mut builders = schema.create_array_builders(8);
1588            for (builder, datum) in builders.iter_mut().zip_eq_fast(datums.iter()) {
1589                builder.append(datum.clone());
1590            }
1591            let columns = builders
1592                .into_iter()
1593                .map(|builder| builder.finish().into())
1594                .collect();
1595            // one row chunk
1596            let chunk = StreamChunk::from_parts(vec![Op::Insert], DataChunk::new(columns, 1));
1597
1598            tx.push_chunk(chunk);
1599        }
1600        let _first_barrier = s.next().await.unwrap();
1601        let upstream_change_log = s.next().await.unwrap().unwrap();
1602        let Message::Chunk(chunk) = upstream_change_log else {
1603            panic!("expect chunk");
1604        };
1605        assert_eq!(chunk.columns().len(), 3);
1606        assert_eq!(chunk.rows().count(), 1);
1607        assert_eq!(
1608            chunk.columns()[0].as_int64().iter().collect::<Vec<_>>(),
1609            vec![Some(44485)]
1610        );
1611        assert_eq!(
1612            chunk.columns()[1].as_int64().iter().collect::<Vec<_>>(),
1613            vec![Some(5)]
1614        );
1615        assert_eq!(
1616            chunk.columns()[2].as_int64().iter().collect::<Vec<_>>(),
1617            vec![Some(100)]
1618        );
1619
1620        drop(tx);
1621        assert!(s.next().await.is_none());
1622    }
1623
1624    #[test]
1625    fn test_reader_startup_polling_selection() {
1626        let recovered_position = OwnedRow::new(vec![Some(ScalarImpl::Int64(1))]);
1627
1628        assert!(!can_poll_upstream_while_creating_reader(
1629            Some(&recovered_position),
1630            false,
1631        ));
1632        assert!(can_poll_upstream_while_creating_reader(None, false));
1633        assert!(can_poll_upstream_while_creating_reader(
1634            Some(&recovered_position),
1635            true,
1636        ));
1637    }
1638
1639    #[tokio::test]
1640    async fn test_snapshot_read_error_rebuilds_reader() {
1641        let external_table = create_mock_external_table().with_mock_snapshot_errors([1, 0]);
1642        let reader_create_probe = external_table.clone();
1643        let (tx, executor) = create_mock_cdc_backfill(
1644            external_table,
1645            create_cdc_state_table(MemoryStateStore::new()).await,
1646            CdcScanOptions {
1647                snapshot_barrier_interval: 1,
1648                ..Default::default()
1649            },
1650        );
1651        let executor = executor.execute_inner();
1652        pin_mut!(executor);
1653
1654        tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
1655        assert!(matches!(
1656            executor.next().await.unwrap().unwrap(),
1657            Message::Barrier(_)
1658        ));
1659
1660        // The second barrier starts the snapshot. The first reader fails when it is polled.
1661        tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
1662        assert!(matches!(
1663            executor.next().await.unwrap().unwrap(),
1664            Message::Barrier(_)
1665        ));
1666
1667        assert!(
1668            tokio::time::timeout(Duration::from_millis(20), executor.next())
1669                .await
1670                .is_err()
1671        );
1672        assert_eq!(reader_create_probe.mock_reader_create_count(), 1);
1673
1674        // The next barrier checkpoints the resume position before the failed reader is replaced.
1675        tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
1676        assert!(matches!(
1677            executor.next().await.unwrap().unwrap(),
1678            Message::Barrier(_)
1679        ));
1680
1681        // The replacement reader succeeds, proving that retries do not reuse the failed reader.
1682        let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
1683            panic!("replacement reader should produce a snapshot chunk");
1684        };
1685        assert_eq!(chunk.cardinality(), 6);
1686        assert_eq!(reader_create_probe.mock_reader_create_count(), 2);
1687
1688        tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
1689        assert!(matches!(
1690            executor.next().await.unwrap().unwrap(),
1691            Message::Barrier(_)
1692        ));
1693    }
1694
1695    #[tokio::test]
1696    async fn test_snapshot_read_error_before_first_chunk_survives_recovery() {
1697        let scan_options = || CdcScanOptions {
1698            snapshot_barrier_interval: 1,
1699            ..Default::default()
1700        };
1701
1702        let memory_state_store = MemoryStateStore::new();
1703
1704        // The first snapshot read fails before any row is emitted, and the next barrier
1705        // checkpoints while there is no snapshot cursor.
1706        {
1707            let (tx, executor) = create_mock_cdc_backfill(
1708                create_mock_external_table().with_mock_snapshot_errors([1]),
1709                create_cdc_state_table(memory_state_store.clone()).await,
1710                scan_options(),
1711            );
1712            let executor = executor.execute_inner();
1713
1714            pin_mut!(executor);
1715
1716            for epoch in 1..=3 {
1717                tx.send_barrier(Barrier::new_test_barrier(test_epoch(epoch)));
1718                assert!(matches!(
1719                    executor.next().await.unwrap().unwrap(),
1720                    Message::Barrier(_)
1721                ));
1722            }
1723        }
1724
1725        let mut restored_state = CdcBackfillState::new(
1726            TableId::new(1234),
1727            create_cdc_state_table(memory_state_store.clone()).await,
1728            5,
1729        );
1730        restored_state
1731            .init_epoch(Barrier::new_test_barrier(test_epoch(4)).epoch)
1732            .await
1733            .unwrap();
1734
1735        // No snapshot progress was made, so no backfill state should be persisted.
1736        let state = restored_state.restore_state().await.unwrap();
1737        assert!(!state.is_finished);
1738        assert_eq!(state.current_pk_pos, None);
1739        assert_eq!(state.last_cdc_offset, None);
1740        assert_eq!(state.row_count, 0);
1741
1742        // After recovery, the snapshot must restart from the beginning instead of resuming
1743        // from a NULL cursor.
1744        let (tx, executor) = create_mock_cdc_backfill(
1745            create_mock_external_table(),
1746            create_cdc_state_table(memory_state_store).await,
1747            scan_options(),
1748        );
1749        let executor = executor.execute_inner();
1750
1751        pin_mut!(executor);
1752
1753        for epoch in 4..=5 {
1754            tx.send_barrier(Barrier::new_test_barrier(test_epoch(epoch)));
1755            assert!(matches!(
1756                executor.next().await.unwrap().unwrap(),
1757                Message::Barrier(_)
1758            ));
1759        }
1760        let Message::Chunk(chunk) = tokio::time::timeout(Duration::from_secs(5), executor.next())
1761            .await
1762            .expect("recovered backfill should read the snapshot")
1763            .unwrap()
1764            .unwrap()
1765        else {
1766            panic!("recovered reader should produce a snapshot chunk");
1767        };
1768
1769        assert_eq!(chunk.cardinality(), 6);
1770    }
1771
1772    #[tokio::test]
1773    async fn test_disabled_cdc_backfill_finishes_without_table_reader() {
1774        let actor_context = ActorContext::for_test(1);
1775        let external_storage_table = ExternalStorageTable::new(
1776            TableId::new(1234),
1777            SchemaTableName {
1778                schema_name: "public".to_owned(),
1779                table_name: "orders".to_owned(),
1780            },
1781            "db".to_owned(),
1782            ExternalTableConfig::default(),
1783            ExternalCdcTableType::Unspecified,
1784            Schema::new(vec![Field::with_name(DataType::Int64, "id")]),
1785            vec![OrderType::ascending()],
1786            Some(vec![CdcKeyComparison::Native]),
1787            vec![0],
1788        );
1789        let schema = Schema::new(vec![
1790            Field::unnamed(DataType::Jsonb),
1791            Field::unnamed(DataType::Varchar),
1792            Field::unnamed(DataType::Varchar),
1793        ]);
1794        let stream_key = vec![1];
1795        let (tx, source) = MockSource::channel();
1796        let source = source.into_executor(schema, stream_key);
1797        let output_columns = vec![ColumnDesc::named("id", ColumnId::new(1), DataType::Int64)];
1798        let memory_state_store = MemoryStateStore::new();
1799        let state_table = create_cdc_state_table(memory_state_store.clone()).await;
1800        let cdc = CdcBackfillExecutor::new(
1801            actor_context,
1802            external_storage_table,
1803            source,
1804            vec![0],
1805            output_columns,
1806            None,
1807            StreamingMetrics::unused().into(),
1808            state_table,
1809            None,
1810            CdcScanOptions {
1811                disable_backfill: true,
1812                ..CdcScanOptions::default()
1813            },
1814            BTreeMap::default(),
1815        );
1816        let executor = cdc.execute_inner();
1817        pin_mut!(executor);
1818
1819        tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
1820        assert!(matches!(
1821            executor.next().await.unwrap().unwrap(),
1822            Message::Barrier(_)
1823        ));
1824
1825        tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
1826        assert!(matches!(
1827            executor.next().await.unwrap().unwrap(),
1828            Message::Barrier(_)
1829        ));
1830
1831        let mut restored_state = CdcBackfillState::new(
1832            TableId::new(1234),
1833            create_cdc_state_table(memory_state_store).await,
1834            5,
1835        );
1836        restored_state
1837            .init_epoch(Barrier::new_test_barrier(test_epoch(2)).epoch)
1838            .await
1839            .unwrap();
1840        let state = restored_state.restore_state().await.unwrap();
1841        assert!(state.is_finished);
1842        assert_eq!(state.last_cdc_offset, None);
1843    }
1844
1845    #[tokio::test]
1846    async fn test_unfinished_cdc_backfill_rejects_null_offset_on_restore() {
1847        let memory_state_store = MemoryStateStore::new();
1848        let mut state_writer = CdcBackfillState::new(
1849            TableId::new(1234),
1850            create_cdc_state_table(memory_state_store.clone()).await,
1851            5,
1852        );
1853        state_writer
1854            .init_epoch(Barrier::new_test_barrier(test_epoch(1)).epoch)
1855            .await
1856            .unwrap();
1857        state_writer
1858            .mutate_and_commit_state(
1859                Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(10))])),
1860                None,
1861                10,
1862                false,
1863                Barrier::new_test_barrier(test_epoch(2)).epoch,
1864            )
1865            .await
1866            .unwrap();
1867
1868        let mut restored_state = CdcBackfillState::new(
1869            TableId::new(1234),
1870            create_cdc_state_table(memory_state_store).await,
1871            5,
1872        );
1873        restored_state
1874            .init_epoch(Barrier::new_test_barrier(test_epoch(2)).epoch)
1875            .await
1876            .unwrap();
1877
1878        assert!(restored_state.restore_state().await.is_err());
1879    }
1880
1881    fn create_raw_cdc_chunk(rows: &[(&str, &str)]) -> StreamChunk {
1882        let schema = Schema::new(vec![
1883            Field::unnamed(DataType::Jsonb),
1884            Field::unnamed(DataType::Varchar),
1885        ]);
1886        let mut builders = schema.create_array_builders(rows.len());
1887        for (payload, offset) in rows {
1888            let payload_datum: Datum = Some(JsonbVal::from_str(payload).unwrap().into());
1889            let offset_datum: Datum = Some((*offset).into());
1890            builders[0].append(payload_datum);
1891            builders[1].append(offset_datum);
1892        }
1893        let columns = builders
1894            .into_iter()
1895            .map(|builder| builder.finish().into())
1896            .collect();
1897        StreamChunk::from_parts(
1898            vec![Op::Insert; rows.len()],
1899            DataChunk::new(columns, rows.len()),
1900        )
1901    }
1902
1903    fn create_mock_external_table() -> ExternalStorageTable {
1904        ExternalStorageTable::new(
1905            TableId::new(1234),
1906            SchemaTableName {
1907                schema_name: "public".to_owned(),
1908                table_name: "mock_table".to_owned(),
1909            },
1910            "mydb".to_owned(),
1911            ExternalTableConfig::default(),
1912            ExternalCdcTableType::Mock,
1913            Schema::new(vec![
1914                Field::with_name(DataType::Int64, "id"),
1915                Field::with_name(DataType::Float64, "price"),
1916            ]),
1917            vec![OrderType::ascending()],
1918            Some(vec![CdcKeyComparison::Native]),
1919            vec![0],
1920        )
1921    }
1922
1923    /// Creates a CDC backfill executor over the `id`/`price` mock table, along with the sender
1924    /// for its upstream.
1925    fn create_mock_cdc_backfill(
1926        external_table: ExternalStorageTable,
1927        state_table: StateTable<MemoryStateStore>,
1928        scan_options: CdcScanOptions,
1929    ) -> (MessageSender, CdcBackfillExecutor<MemoryStateStore>) {
1930        let (tx, source) = MockSource::channel();
1931        let source = source.into_executor(
1932            Schema::new(vec![
1933                Field::unnamed(DataType::Jsonb),
1934                Field::unnamed(DataType::Varchar),
1935            ]),
1936            vec![0],
1937        );
1938        let executor = CdcBackfillExecutor::new(
1939            ActorContext::for_test(0x1a),
1940            external_table,
1941            source,
1942            vec![0, 1],
1943            vec![
1944                ColumnDesc::named("id", ColumnId::new(1), DataType::Int64),
1945                ColumnDesc::named("price", ColumnId::new(2), DataType::Float64),
1946            ],
1947            None,
1948            StreamingMetrics::unused().into(),
1949            state_table,
1950            None,
1951            scan_options,
1952            BTreeMap::default(),
1953        );
1954        (tx, executor)
1955    }
1956
1957    async fn create_cdc_state_table(store: MemoryStateStore) -> StateTable<MemoryStateStore> {
1958        let state_schema = Schema::new(vec![
1959            Field::with_name(DataType::Varchar, "split_id"),
1960            Field::with_name(DataType::Int64, "id"),
1961            Field::with_name(DataType::Boolean, "backfill_finished"),
1962            Field::with_name(DataType::Int64, "row_count"),
1963            Field::with_name(DataType::Jsonb, "cdc_offset"),
1964        ]);
1965        let column_descs = vec![
1966            ColumnDesc::unnamed(ColumnId::from(0), state_schema[0].data_type.clone()),
1967            ColumnDesc::unnamed(ColumnId::from(1), state_schema[1].data_type.clone()),
1968            ColumnDesc::unnamed(ColumnId::from(2), state_schema[2].data_type.clone()),
1969            ColumnDesc::unnamed(ColumnId::from(3), state_schema[3].data_type.clone()),
1970            ColumnDesc::unnamed(ColumnId::from(4), state_schema[4].data_type.clone()),
1971        ];
1972
1973        StateTable::from_table_catalog(
1974            &gen_pbtable(
1975                TableId::from(0x42),
1976                column_descs,
1977                vec![OrderType::ascending()],
1978                vec![0],
1979                0,
1980            ),
1981            store,
1982            None,
1983        )
1984        .await
1985    }
1986
1987    async fn create_recovering_cdc_backfill() -> (
1988        MessageSender,
1989        CdcBackfillExecutor<MemoryStateStore>,
1990        MemoryStateStore,
1991    ) {
1992        let memory_state_store = MemoryStateStore::new();
1993        let mut state_writer = CdcBackfillState::new(
1994            TableId::new(1234),
1995            create_cdc_state_table(memory_state_store.clone()).await,
1996            5,
1997        );
1998        state_writer
1999            .init_epoch(Barrier::new_test_barrier(test_epoch(1)).epoch)
2000            .await
2001            .unwrap();
2002        state_writer
2003            .mutate_and_commit_state(
2004                Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2005                Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2006                5,
2007                false,
2008                Barrier::new_test_barrier(test_epoch(2)).epoch,
2009            )
2010            .await
2011            .unwrap();
2012
2013        let (tx, executor) = create_mock_cdc_backfill(
2014            create_mock_external_table(),
2015            create_cdc_state_table(memory_state_store.clone()).await,
2016            CdcScanOptions::default(),
2017        );
2018        (tx, executor, memory_state_store)
2019    }
2020
2021    async fn assert_recovery_offset(
2022        memory_state_store: MemoryStateStore,
2023        expected_position: u64,
2024        epoch: u64,
2025    ) {
2026        let mut restored_state = CdcBackfillState::new(
2027            TableId::new(1234),
2028            create_cdc_state_table(memory_state_store).await,
2029            5,
2030        );
2031        restored_state
2032            .init_epoch(Barrier::new_test_barrier(epoch).epoch)
2033            .await
2034            .unwrap();
2035        let state = restored_state.restore_state().await.unwrap();
2036        assert_eq!(
2037            state.last_cdc_offset,
2038            Some(CdcOffset::Mysql(MysqlOffset::new(
2039                "1.binlog".to_owned(),
2040                expected_position,
2041            )))
2042        );
2043    }
2044
2045    #[tokio::test]
2046    async fn test_recovery_filters_upstream_while_reader_is_pending() {
2047        let chunk = StreamChunk::from_rows(
2048            &[
2049                (
2050                    Op::Insert,
2051                    OwnedRow::new(vec![
2052                        Some(ScalarImpl::Int64(4)),
2053                        Some(ScalarImpl::Float64(44.04.into())),
2054                        Some(ScalarImpl::Utf8(
2055                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2056                                .into(),
2057                        )),
2058                    ]),
2059                ),
2060                (
2061                    Op::Insert,
2062                    OwnedRow::new(vec![
2063                        Some(ScalarImpl::Int64(6)),
2064                        Some(ScalarImpl::Float64(66.06.into())),
2065                        Some(ScalarImpl::Utf8(
2066                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2067                                .into(),
2068                        )),
2069                    ]),
2070                ),
2071            ],
2072            &[DataType::Int64, DataType::Float64, DataType::Varchar],
2073        );
2074        let mut upstream = stream::iter([
2075            Ok(Message::Chunk(chunk)),
2076            Ok(Message::Barrier(Barrier::new_test_barrier(test_epoch(4)))),
2077        ]);
2078        let mut reader_future = Box::pin(std::future::pending::<ExternalTableReaderImpl>());
2079
2080        let Either::Left(Message::Chunk(chunk)) =
2081            build_reader_and_poll_upstream(&mut upstream, &mut reader_future)
2082                .await
2083                .unwrap()
2084        else {
2085            panic!("expected an upstream chunk while the reader is pending");
2086        };
2087
2088        let parser = ExternalCdcTableType::Mock.get_cdc_offset_parser().unwrap();
2089        let (chunk, consumed_offset) =
2090            CdcBackfillExecutor::<MemoryStateStore>::filter_recovery_chunk(
2091                &parser,
2092                chunk,
2093                &OwnedRow::new(vec![Some(ScalarImpl::Int64(5))]),
2094                PkCompareInfo {
2095                    indices: &[0],
2096                    order: &[OrderType::ascending()],
2097                    needs_unsigned_i64_compare: &[false],
2098                },
2099                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2100                &[0, 1],
2101            )
2102            .unwrap();
2103        let chunk = chunk.expect("the scanned row should be emitted");
2104        assert_eq!(chunk.cardinality(), 1);
2105        assert_eq!(
2106            chunk.row_at(0).1.to_owned_row()[0],
2107            Some(ScalarImpl::Int64(4))
2108        );
2109        assert_eq!(
2110            consumed_offset,
2111            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 4)))
2112        );
2113        assert!(matches!(
2114            build_reader_and_poll_upstream(&mut upstream, &mut reader_future)
2115                .await
2116                .unwrap(),
2117            Either::Left(Message::Barrier(_))
2118        ));
2119    }
2120
2121    #[tokio::test]
2122    async fn test_recovery_filters_updates_before_backfill_barrier() {
2123        let (mut tx, executor, memory_state_store) = create_recovering_cdc_backfill().await;
2124        let executor = executor.execute_inner();
2125        pin_mut!(executor);
2126
2127        tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2128        assert!(matches!(
2129            executor.next().await.unwrap().unwrap(),
2130            Message::Barrier(_)
2131        ));
2132
2133        tx.push_chunk(create_raw_cdc_chunk(&[
2134            (
2135                r#"{ "payload": { "before": { "id": 1, "price": 11.01 }, "after": { "id": 1, "price": 12.01 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "u", "ts_ms": 1695277757017, "transaction": null } }"#,
2136                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2137            ),
2138            (
2139                r#"{ "payload": { "before": { "id": 6, "price": 66.06 }, "after": { "id": 6, "price": 67.06 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "u", "ts_ms": 1695277757017, "transaction": null } }"#,
2140                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2141            ),
2142        ]));
2143        tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2144
2145        let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
2146            panic!("expected the recovered update before the barrier");
2147        };
2148        let rows = chunk
2149            .rows()
2150            .map(|(op, row)| (op, row.to_owned_row()))
2151            .collect::<Vec<_>>();
2152        assert_eq!(rows.len(), 1);
2153        assert_eq!(rows[0].0, Op::Insert);
2154        assert_eq!(rows[0].1[0], Some(ScalarImpl::Int64(1)));
2155        assert!(matches!(
2156            executor.next().await.unwrap().unwrap(),
2157            Message::Barrier(_)
2158        ));
2159        assert_recovery_offset(memory_state_store, 4, test_epoch(4)).await;
2160    }
2161
2162    #[tokio::test]
2163    async fn test_recovery_preserves_scanned_inserts_and_deletes() {
2164        let (mut tx, executor, _) = create_recovering_cdc_backfill().await;
2165        let executor = executor.execute_inner();
2166        pin_mut!(executor);
2167
2168        tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2169        assert!(matches!(
2170            executor.next().await.unwrap().unwrap(),
2171            Message::Barrier(_)
2172        ));
2173        // This replayed event predates the saved offset and must not lower the offset filter.
2174        tx.push_chunk(create_raw_cdc_chunk(&[(
2175            r#"{ "payload": { "before": null, "after": { "id": 3, "price": 30.01 }, "op": "c" } }"#,
2176            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":1},"isHeartbeat":false}"#,
2177        )]));
2178        tx.push_chunk(create_raw_cdc_chunk(&[
2179            (
2180                r#"{ "payload": { "before": null, "after": { "id": 3, "price": 31.01 }, "op": "c" } }"#,
2181                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":1},"isHeartbeat":false}"#,
2182            ),
2183            (
2184                r#"{ "payload": { "before": null, "after": { "id": 4, "price": 44.04 }, "op": "c" } }"#,
2185                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":2},"isHeartbeat":false}"#,
2186            ),
2187            (
2188                r#"{ "payload": { "before": { "id": 5, "price": 55.05 }, "after": null, "op": "d" } }"#,
2189                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2190            ),
2191        ]));
2192        tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2193
2194        let Message::Chunk(chunk) = executor.next().await.unwrap().unwrap() else {
2195            panic!("expected the scanned-prefix changes before the barrier");
2196        };
2197        let rows = chunk
2198            .rows()
2199            .map(|(op, row)| (op, row.to_owned_row()))
2200            .collect::<Vec<_>>();
2201        assert_eq!(rows.len(), 2);
2202        assert_eq!(rows[0].0, Op::Insert);
2203        assert_eq!(rows[0].1[0], Some(ScalarImpl::Int64(4)));
2204        assert_eq!(rows[1].0, Op::Delete);
2205        assert_eq!(rows[1].1[0], Some(ScalarImpl::Int64(5)));
2206        assert!(matches!(
2207            executor.next().await.unwrap().unwrap(),
2208            Message::Barrier(_)
2209        ));
2210    }
2211
2212    #[tokio::test]
2213    async fn test_recovery_does_not_leave_deleted_unscanned_row() {
2214        let (mut tx, mut executor, memory_state_store) = create_recovering_cdc_backfill().await;
2215        executor.options.snapshot_barrier_interval = 1;
2216        let executor = executor.execute_inner();
2217        pin_mut!(executor);
2218
2219        tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2220        assert!(matches!(
2221            executor.next().await.unwrap().unwrap(),
2222            Message::Barrier(_)
2223        ));
2224
2225        // Recover at PK 5, then receive an insert for an unscanned key during startup.
2226        tx.push_chunk(create_raw_cdc_chunk(&[(
2227            r#"{ "payload": { "before": null, "after": { "id": 100, "price": 100.01 }, "op": "c" } }"#,
2228            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2229        )]));
2230        tx.send_barrier(Barrier::new_test_barrier(test_epoch(4)));
2231
2232        // Normal backfill receives its delete before the snapshot reaches PK 100.
2233        // The snapshot refresh at barrier 5 scans up to PK 8 and filters out the delete.
2234        tx.push_chunk(create_raw_cdc_chunk(&[(
2235            r#"{ "payload": { "before": { "id": 100, "price": 100.01 }, "after": null, "op": "d" } }"#,
2236            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2237        )]));
2238        tx.send_barrier(Barrier::new_test_barrier(test_epoch(5)));
2239        // The next snapshot is empty, so backfill completes at barrier 6.
2240        tx.send_barrier(Barrier::new_test_barrier(test_epoch(6)));
2241
2242        let mut materialized = BTreeMap::new();
2243        loop {
2244            match executor.next().await.unwrap().unwrap() {
2245                Message::Chunk(chunk) => {
2246                    for (op, row) in chunk.rows() {
2247                        let pk = row.datum_at(0).unwrap().into_int64();
2248                        match op {
2249                            Op::Insert => {
2250                                materialized.insert(pk, row.to_owned_row());
2251                            }
2252                            Op::Delete => {
2253                                materialized.remove(&pk);
2254                            }
2255                            _ => unreachable!("CDC updates are converted to inserts"),
2256                        }
2257                    }
2258                }
2259                Message::Barrier(barrier) if barrier.epoch.curr == test_epoch(6) => break,
2260                Message::Barrier(_) => {}
2261                Message::Watermark(_) => panic!("unexpected watermark during backfill"),
2262            }
2263        }
2264
2265        assert!(!materialized.contains_key(&100));
2266        let mut restored_state = CdcBackfillState::new(
2267            TableId::new(1234),
2268            create_cdc_state_table(memory_state_store).await,
2269            5,
2270        );
2271        restored_state
2272            .init_epoch(Barrier::new_test_barrier(test_epoch(6)).epoch)
2273            .await
2274            .unwrap();
2275        assert!(restored_state.restore_state().await.unwrap().is_finished);
2276    }
2277
2278    #[test]
2279    fn test_consume_upstream_chunk_buffer_retains_future_rows() {
2280        let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2281            &[
2282                (
2283                    Op::Insert,
2284                    OwnedRow::new(vec![
2285                        Some(ScalarImpl::Int64(1)),
2286                        Some(ScalarImpl::Int64(100)),
2287                        Some(ScalarImpl::Utf8(
2288                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2289                                .into(),
2290                        )),
2291                    ]),
2292                ),
2293                (
2294                    Op::Insert,
2295                    OwnedRow::new(vec![
2296                        Some(ScalarImpl::Int64(6)),
2297                        Some(ScalarImpl::Int64(600)),
2298                        Some(ScalarImpl::Utf8(
2299                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2300                                .into(),
2301                        )),
2302                    ]),
2303                ),
2304            ],
2305            &[DataType::Int64, DataType::Int64, DataType::Varchar],
2306        )];
2307
2308        let (emitted_chunks, drained_row_count, drained_offset) =
2309            CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2310                &MockExternalTableReader::get_cdc_offset_parser(),
2311                &mut upstream_chunk_buffer,
2312                Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2313                PkCompareInfo {
2314                    indices: &[0],
2315                    order: &[OrderType::ascending()],
2316                    needs_unsigned_i64_compare: &[false],
2317                },
2318                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2319                &[0, 1],
2320            )
2321            .unwrap();
2322
2323        assert_eq!(drained_row_count, 1);
2324        assert_eq!(
2325            drained_offset,
2326            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2327        );
2328        assert_eq!(emitted_chunks.len(), 1);
2329        assert_eq!(emitted_chunks[0].rows().count(), 1);
2330        assert_eq!(
2331            emitted_chunks[0].rows().next().unwrap().1.to_owned_row(),
2332            OwnedRow::new(vec![
2333                Some(ScalarImpl::Int64(1)),
2334                Some(ScalarImpl::Int64(100))
2335            ])
2336        );
2337
2338        assert_eq!(upstream_chunk_buffer.len(), 1);
2339        assert_eq!(upstream_chunk_buffer[0].rows().count(), 1);
2340        assert_eq!(
2341            upstream_chunk_buffer[0]
2342                .rows()
2343                .next()
2344                .unwrap()
2345                .1
2346                .to_owned_row(),
2347            OwnedRow::new(vec![
2348                Some(ScalarImpl::Int64(6)),
2349                Some(ScalarImpl::Int64(600)),
2350                Some(ScalarImpl::Utf8(
2351                    r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2352                        .into(),
2353                )),
2354            ])
2355        );
2356    }
2357
2358    #[test]
2359    fn test_consume_buffer_uses_unsigned_bigint_pk_order() {
2360        let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2361            &[
2362                (
2363                    Op::Insert,
2364                    OwnedRow::new(vec![
2365                        Some(ScalarImpl::Int64(4)),
2366                        Some(ScalarImpl::Int64(400)),
2367                        Some(ScalarImpl::Utf8(
2368                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2369                                .into(),
2370                        )),
2371                    ]),
2372                ),
2373                (
2374                    Op::Insert,
2375                    OwnedRow::new(vec![
2376                        // `u64::MAX` represented in RisingWave's `i64` storage.
2377                        Some(ScalarImpl::Int64(-1)),
2378                        Some(ScalarImpl::Int64(900)),
2379                        Some(ScalarImpl::Utf8(
2380                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2381                                .into(),
2382                        )),
2383                    ]),
2384                ),
2385            ],
2386            &[DataType::Int64, DataType::Int64, DataType::Varchar],
2387        )];
2388
2389        let (emitted_chunks, drained_row_count, drained_offset) =
2390            CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2391                &MockExternalTableReader::get_cdc_offset_parser(),
2392                &mut upstream_chunk_buffer,
2393                Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2394                PkCompareInfo {
2395                    indices: &[0],
2396                    order: &[OrderType::ascending()],
2397                    needs_unsigned_i64_compare: &[true],
2398                },
2399                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2400                &[0, 1],
2401            )
2402            .unwrap();
2403
2404        assert_eq!(drained_row_count, 1);
2405        assert_eq!(
2406            drained_offset,
2407            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2408        );
2409        assert_eq!(emitted_chunks.len(), 1);
2410        assert_eq!(
2411            emitted_chunks[0].rows().next().unwrap().1.to_owned_row(),
2412            OwnedRow::new(vec![
2413                Some(ScalarImpl::Int64(4)),
2414                Some(ScalarImpl::Int64(400))
2415            ])
2416        );
2417
2418        assert_eq!(upstream_chunk_buffer.len(), 1);
2419        assert_eq!(
2420            upstream_chunk_buffer[0]
2421                .rows()
2422                .next()
2423                .unwrap()
2424                .1
2425                .to_owned_row(),
2426            OwnedRow::new(vec![
2427                Some(ScalarImpl::Int64(-1)),
2428                Some(ScalarImpl::Int64(900)),
2429                Some(ScalarImpl::Utf8(
2430                    r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2431                        .into(),
2432                )),
2433            ])
2434        );
2435    }
2436
2437    #[test]
2438    fn test_consume_buffer_non_monotonic_pk_in_chunk() {
2439        let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2440            &[
2441                (
2442                    Op::Insert,
2443                    OwnedRow::new(vec![
2444                        Some(ScalarImpl::Int64(1)),
2445                        Some(ScalarImpl::Int64(100)),
2446                        Some(ScalarImpl::Utf8(
2447                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2448                                .into(),
2449                        )),
2450                    ]),
2451                ),
2452                (
2453                    Op::Insert,
2454                    OwnedRow::new(vec![
2455                        Some(ScalarImpl::Int64(6)),
2456                        Some(ScalarImpl::Int64(600)),
2457                        Some(ScalarImpl::Utf8(
2458                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2459                                .into(),
2460                        )),
2461                    ]),
2462                ),
2463                (
2464                    Op::Insert,
2465                    OwnedRow::new(vec![
2466                        Some(ScalarImpl::Int64(2)),
2467                        Some(ScalarImpl::Int64(200)),
2468                        Some(ScalarImpl::Utf8(
2469                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2470                                .into(),
2471                        )),
2472                    ]),
2473                ),
2474            ],
2475            &[DataType::Int64, DataType::Int64, DataType::Varchar],
2476        )];
2477
2478        let (emitted_chunks, drained_row_count, drained_offset) =
2479            CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2480                &MockExternalTableReader::get_cdc_offset_parser(),
2481                &mut upstream_chunk_buffer,
2482                Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2483                PkCompareInfo {
2484                    indices: &[0],
2485                    order: &[OrderType::ascending()],
2486                    needs_unsigned_i64_compare: &[false],
2487                },
2488                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2489                &[0, 1],
2490            )
2491            .unwrap();
2492
2493        assert_eq!(drained_row_count, 2);
2494        assert_eq!(
2495            drained_offset,
2496            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2497        );
2498        assert_eq!(emitted_chunks.len(), 1);
2499        assert_eq!(emitted_chunks[0].rows().count(), 2);
2500        assert_eq!(
2501            emitted_chunks[0]
2502                .rows()
2503                .map(|(_, row)| row.to_owned_row())
2504                .collect::<Vec<_>>(),
2505            vec![
2506                OwnedRow::new(vec![
2507                    Some(ScalarImpl::Int64(1)),
2508                    Some(ScalarImpl::Int64(100))
2509                ]),
2510                OwnedRow::new(vec![
2511                    Some(ScalarImpl::Int64(2)),
2512                    Some(ScalarImpl::Int64(200))
2513                ]),
2514            ]
2515        );
2516
2517        assert_eq!(upstream_chunk_buffer.len(), 1);
2518        assert_eq!(upstream_chunk_buffer[0].rows().count(), 1);
2519        assert_eq!(
2520            upstream_chunk_buffer[0]
2521                .rows()
2522                .next()
2523                .unwrap()
2524                .1
2525                .to_owned_row(),
2526            OwnedRow::new(vec![
2527                Some(ScalarImpl::Int64(6)),
2528                Some(ScalarImpl::Int64(600)),
2529                Some(ScalarImpl::Utf8(
2530                    r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2531                        .into(),
2532                )),
2533            ])
2534        );
2535    }
2536
2537    #[test]
2538    fn test_consume_buffer_processes_chunks_after_future_row() {
2539        let mut upstream_chunk_buffer = vec![
2540            StreamChunk::from_rows(
2541                &[
2542                    (
2543                        Op::Insert,
2544                        OwnedRow::new(vec![
2545                            Some(ScalarImpl::Int64(1)),
2546                            Some(ScalarImpl::Int64(100)),
2547                            Some(ScalarImpl::Utf8(
2548                                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#
2549                                    .into(),
2550                            )),
2551                        ]),
2552                    ),
2553                    (
2554                        Op::Insert,
2555                        OwnedRow::new(vec![
2556                            Some(ScalarImpl::Int64(6)),
2557                            Some(ScalarImpl::Int64(600)),
2558                            Some(ScalarImpl::Utf8(
2559                                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2560                                    .into(),
2561                            )),
2562                        ]),
2563                    ),
2564                ],
2565                &[DataType::Int64, DataType::Int64, DataType::Varchar],
2566            ),
2567            StreamChunk::from_rows(
2568                &[(
2569                    Op::Insert,
2570                    OwnedRow::new(vec![
2571                        Some(ScalarImpl::Int64(2)),
2572                        Some(ScalarImpl::Int64(200)),
2573                        Some(ScalarImpl::Utf8(
2574                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2575                                .into(),
2576                        )),
2577                    ]),
2578                )],
2579                &[DataType::Int64, DataType::Int64, DataType::Varchar],
2580            ),
2581        ];
2582
2583        let (emitted_chunks, drained_row_count, drained_offset) =
2584            CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2585                &MockExternalTableReader::get_cdc_offset_parser(),
2586                &mut upstream_chunk_buffer,
2587                Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(5))])),
2588                PkCompareInfo {
2589                    indices: &[0],
2590                    order: &[OrderType::ascending()],
2591                    needs_unsigned_i64_compare: &[false],
2592                },
2593                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 2))),
2594                &[0, 1],
2595            )
2596            .unwrap();
2597
2598        assert_eq!(drained_row_count, 2);
2599        assert_eq!(
2600            drained_offset,
2601            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3)))
2602        );
2603        assert_eq!(emitted_chunks.len(), 2);
2604        assert_eq!(emitted_chunks[0].rows().count(), 1);
2605        assert_eq!(emitted_chunks[1].rows().count(), 1);
2606        assert_eq!(
2607            emitted_chunks[1].rows().next().unwrap().1.to_owned_row(),
2608            OwnedRow::new(vec![
2609                Some(ScalarImpl::Int64(2)),
2610                Some(ScalarImpl::Int64(200))
2611            ])
2612        );
2613
2614        assert_eq!(upstream_chunk_buffer.len(), 1);
2615        assert_eq!(
2616            upstream_chunk_buffer[0]
2617                .rows()
2618                .next()
2619                .unwrap()
2620                .1
2621                .to_owned_row(),
2622            OwnedRow::new(vec![
2623                Some(ScalarImpl::Int64(6)),
2624                Some(ScalarImpl::Int64(600)),
2625                Some(ScalarImpl::Utf8(
2626                    r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2627                        .into(),
2628                )),
2629            ])
2630        );
2631    }
2632
2633    #[test]
2634    fn test_consume_buffer_advances_offset_after_preceding_rows_are_emitted() {
2635        let mut upstream_chunk_buffer = vec![StreamChunk::from_rows(
2636            &[
2637                (
2638                    Op::Insert,
2639                    OwnedRow::new(vec![
2640                        Some(ScalarImpl::Int64(6)),
2641                        Some(ScalarImpl::Int64(600)),
2642                        Some(ScalarImpl::Utf8(
2643                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#
2644                                .into(),
2645                        )),
2646                    ]),
2647                ),
2648                (
2649                    Op::Insert,
2650                    OwnedRow::new(vec![
2651                        Some(ScalarImpl::Int64(2)),
2652                        Some(ScalarImpl::Int64(200)),
2653                        Some(ScalarImpl::Utf8(
2654                            r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":5},"isHeartbeat":false}"#
2655                                .into(),
2656                        )),
2657                    ]),
2658                ),
2659            ],
2660            &[DataType::Int64, DataType::Int64, DataType::Varchar],
2661        )];
2662
2663        let (_, drained_row_count, drained_offset) =
2664            CdcBackfillExecutor::<MemoryStateStore>::consume_upstream_chunk_buffer(
2665                &MockExternalTableReader::get_cdc_offset_parser(),
2666                &mut upstream_chunk_buffer,
2667                Some(&OwnedRow::new(vec![Some(ScalarImpl::Int64(6))])),
2668                PkCompareInfo {
2669                    indices: &[0],
2670                    order: &[OrderType::ascending()],
2671                    needs_unsigned_i64_compare: &[false],
2672                },
2673                &Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 3))),
2674                &[0, 1],
2675            )
2676            .unwrap();
2677
2678        assert_eq!(drained_row_count, 2);
2679        assert_eq!(
2680            drained_offset,
2681            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 5)))
2682        );
2683        assert!(upstream_chunk_buffer.is_empty());
2684    }
2685
2686    #[tokio::test]
2687    async fn test_cdc_backfill_persists_buffered_offset_on_checkpoint() {
2688        let memory_state_store = MemoryStateStore::new();
2689        let state_table = create_cdc_state_table(memory_state_store.clone()).await;
2690
2691        let (mut tx, executor) = create_mock_cdc_backfill(
2692            create_mock_external_table(),
2693            state_table,
2694            CdcScanOptions {
2695                snapshot_barrier_interval: 10,
2696                ..Default::default()
2697            },
2698        );
2699        let executor = executor.execute_inner();
2700        pin_mut!(executor);
2701
2702        tx.send_barrier(Barrier::new_test_barrier(test_epoch(1)));
2703        assert!(matches!(
2704            executor.next().await.unwrap().unwrap(),
2705            Message::Barrier(_)
2706        ));
2707
2708        tx.send_barrier(Barrier::new_test_barrier(test_epoch(2)));
2709        assert!(matches!(
2710            executor.next().await.unwrap().unwrap(),
2711            Message::Barrier(_)
2712        ));
2713
2714        assert!(matches!(
2715            executor.next().await.unwrap().unwrap(),
2716            Message::Chunk(_)
2717        ));
2718
2719        tx.push_chunk(create_raw_cdc_chunk(&[
2720            (
2721                r#"{ "payload": { "before": null, "after": { "id": 1, "price": 10.01 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
2722                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":3},"isHeartbeat":false}"#,
2723            ),
2724            (
2725                r#"{ "payload": { "before": null, "after": { "id": 6, "price": 66.06 }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002" }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
2726                r#"{"sourcePartition":{},"sourceOffset":{"file":"1.binlog","pos":4},"isHeartbeat":false}"#,
2727            ),
2728        ]));
2729        tx.send_barrier(Barrier::new_test_barrier(test_epoch(3)));
2730
2731        assert!(matches!(
2732            executor.next().await.unwrap().unwrap(),
2733            Message::Chunk(_)
2734        ));
2735        assert!(matches!(
2736            executor.next().await.unwrap().unwrap(),
2737            Message::Barrier(_)
2738        ));
2739
2740        let mut restored_state = CdcBackfillState::new(
2741            TableId::new(1234),
2742            create_cdc_state_table(memory_state_store).await,
2743            5,
2744        );
2745        restored_state
2746            .init_epoch(Barrier::new_test_barrier(test_epoch(3)).epoch)
2747            .await
2748            .unwrap();
2749        let state = restored_state.restore_state().await.unwrap();
2750        assert_eq!(
2751            state.last_cdc_offset,
2752            Some(CdcOffset::Mysql(MysqlOffset::new("1.binlog".to_owned(), 4)))
2753        );
2754    }
2755}