risingwave_frontend/optimizer/plan_node/generic/
cdc_scan.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::rc::Rc;
16use std::str::FromStr;
17
18use anyhow::anyhow;
19use educe::Educe;
20use pretty_xmlish::Pretty;
21use risingwave_common::catalog::{CdcTableDesc, ColumnDesc, Field, Schema};
22use risingwave_common::util::column_index_mapping::ColIndexMapping;
23use risingwave_common::util::sort_util::ColumnOrder;
24use risingwave_connector::source::cdc::external::CdcTableType;
25use risingwave_connector::source::cdc::{
26    CDC_BACKFILL_AS_EVEN_SPLITS, CDC_BACKFILL_ENABLE_KEY, CDC_BACKFILL_MAX_PARALLELISM,
27    CDC_BACKFILL_NUM_ROWS_PER_SPLIT, CDC_BACKFILL_PARALLELISM,
28    CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY, CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY,
29    CDC_BACKFILL_SPLIT_PK_COLUMN_INDEX, CdcScanOptions,
30};
31
32use super::GenericPlanNode;
33use crate::WithOptions;
34use crate::catalog::ColumnId;
35use crate::error::Result;
36use crate::expr::{ExprRewriter, ExprVisitor};
37use crate::optimizer::optimizer_context::OptimizerContextRef;
38use crate::optimizer::property::{FunctionalDependencySet, MonotonicityMap, WatermarkColumns};
39
40/// [`CdcScan`] reads rows of a table from an external upstream database
41#[derive(Debug, Clone, Educe)]
42#[educe(PartialEq, Eq, Hash)]
43pub struct CdcScan {
44    pub table_name: String,
45    /// Include `output_col_idx` and columns required in `predicate`
46    pub output_col_idx: Vec<usize>,
47    /// Descriptor of the external table for CDC
48    pub cdc_table_desc: Rc<CdcTableDesc>,
49    #[educe(PartialEq(ignore))]
50    #[educe(Hash(ignore))]
51    pub ctx: OptimizerContextRef,
52
53    pub options: CdcScanOptions,
54}
55
56pub fn build_cdc_scan_options_with_options(
57    with_options: &WithOptions,
58    cdc_table_type: CdcTableType,
59) -> Result<CdcScanOptions> {
60    // Update this after more CDC table type is supported for backfill v2.
61    let support_backfill_v2 = matches!(cdc_table_type, CdcTableType::Postgres | CdcTableType::Mock);
62
63    // unspecified option will use default values
64    let mut scan_options = CdcScanOptions::default();
65
66    // disable backfill if 'snapshot=false'
67    if let Some(snapshot) = with_options.get(CDC_BACKFILL_ENABLE_KEY) {
68        scan_options.disable_backfill = !(bool::from_str(snapshot)
69            .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_ENABLE_KEY))?);
70    };
71
72    if let Some(snapshot_interval) = with_options.get(CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY) {
73        scan_options.snapshot_barrier_interval = u32::from_str(snapshot_interval)
74            .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY))?;
75    };
76
77    if let Some(snapshot_batch_size) = with_options.get(CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY) {
78        scan_options.snapshot_batch_size = u32::from_str(snapshot_batch_size)
79            .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY))?;
80    };
81
82    if support_backfill_v2 {
83        if let Some(backfill_parallelism) = with_options.get(CDC_BACKFILL_PARALLELISM) {
84            scan_options.backfill_parallelism = u32::from_str(backfill_parallelism)
85                .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_PARALLELISM))?;
86            if scan_options.backfill_parallelism > CDC_BACKFILL_MAX_PARALLELISM {
87                return Err(anyhow!(
88                    "Invalid value for {}, should be in range [0,{}] ",
89                    CDC_BACKFILL_PARALLELISM,
90                    CDC_BACKFILL_MAX_PARALLELISM
91                )
92                .into());
93            }
94        }
95
96        if let Some(backfill_num_rows_per_split) = with_options.get(CDC_BACKFILL_NUM_ROWS_PER_SPLIT)
97        {
98            scan_options.backfill_num_rows_per_split = u64::from_str(backfill_num_rows_per_split)
99                .map_err(|_| {
100                anyhow!("Invalid value for {}", CDC_BACKFILL_NUM_ROWS_PER_SPLIT)
101            })?;
102        }
103
104        if let Some(backfill_as_even_splits) = with_options.get(CDC_BACKFILL_AS_EVEN_SPLITS) {
105            scan_options.backfill_as_even_splits = bool::from_str(backfill_as_even_splits)
106                .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_AS_EVEN_SPLITS))?;
107        }
108
109        if let Some(backfill_split_pk_column_index) =
110            with_options.get(CDC_BACKFILL_SPLIT_PK_COLUMN_INDEX)
111        {
112            scan_options.backfill_split_pk_column_index =
113                u32::from_str(backfill_split_pk_column_index).map_err(|_| {
114                    anyhow!("Invalid value for {}", CDC_BACKFILL_SPLIT_PK_COLUMN_INDEX)
115                })?;
116        }
117    }
118
119    Ok(scan_options)
120}
121
122impl CdcScan {
123    pub fn rewrite_exprs(&self, _rewriter: &mut dyn ExprRewriter) {}
124
125    pub fn visit_exprs(&self, _v: &mut dyn ExprVisitor) {}
126
127    /// Get the ids of the output columns.
128    pub fn output_column_ids(&self) -> Vec<ColumnId> {
129        self.output_col_idx
130            .iter()
131            .map(|i| self.get_table_columns()[*i].column_id)
132            .collect()
133    }
134
135    pub fn primary_key(&self) -> &[ColumnOrder] {
136        &self.cdc_table_desc.pk
137    }
138
139    pub fn watermark_columns(&self) -> WatermarkColumns {
140        WatermarkColumns::new()
141    }
142
143    pub fn columns_monotonicity(&self) -> MonotonicityMap {
144        MonotonicityMap::new()
145    }
146
147    pub(crate) fn column_names_with_table_prefix(&self) -> Vec<String> {
148        self.output_col_idx
149            .iter()
150            .map(|&i| format!("{}.{}", self.table_name, self.get_table_columns()[i].name))
151            .collect()
152    }
153
154    pub(crate) fn column_names(&self) -> Vec<String> {
155        self.output_col_idx
156            .iter()
157            .map(|&i| self.get_table_columns()[i].name.clone())
158            .collect()
159    }
160
161    /// get the Mapping of columnIndex from internal column index to output column index
162    pub fn i2o_col_mapping(&self) -> ColIndexMapping {
163        ColIndexMapping::with_remaining_columns(
164            &self.output_col_idx,
165            self.get_table_columns().len(),
166        )
167    }
168
169    /// Get the ids of the output columns and primary key columns.
170    pub fn output_and_pk_column_ids(&self) -> Vec<ColumnId> {
171        let mut ids = self.output_column_ids();
172        for column_order in self.primary_key() {
173            let id = self.get_table_columns()[column_order.column_index].column_id;
174            if !ids.contains(&id) {
175                ids.push(id);
176            }
177        }
178        ids
179    }
180
181    /// Create a logical scan node for CDC backfill
182    pub(crate) fn new(
183        table_name: String,
184        output_col_idx: Vec<usize>, // the column index in the table
185        cdc_table_desc: Rc<CdcTableDesc>,
186        ctx: OptimizerContextRef,
187        options: CdcScanOptions,
188    ) -> Self {
189        Self {
190            table_name,
191            output_col_idx,
192            cdc_table_desc,
193            ctx,
194            options,
195        }
196    }
197
198    pub(crate) fn columns_pretty<'a>(&self, verbose: bool) -> Pretty<'a> {
199        Pretty::Array(
200            match verbose {
201                true => self.column_names_with_table_prefix(),
202                false => self.column_names(),
203            }
204            .into_iter()
205            .map(Pretty::from)
206            .collect(),
207        )
208    }
209}
210
211// TODO: extend for cdc table
212impl GenericPlanNode for CdcScan {
213    fn schema(&self) -> Schema {
214        let fields = self
215            .output_col_idx
216            .iter()
217            .map(|tb_idx| {
218                let col = &self.get_table_columns()[*tb_idx];
219                Field::from_with_table_name_prefix(col, &self.table_name)
220            })
221            .collect();
222        Schema { fields }
223    }
224
225    fn stream_key(&self) -> Option<Vec<usize>> {
226        Some(self.cdc_table_desc.stream_key.clone())
227    }
228
229    fn ctx(&self) -> OptimizerContextRef {
230        self.ctx.clone()
231    }
232
233    fn functional_dependency(&self) -> FunctionalDependencySet {
234        let pk_indices = self.stream_key();
235        let col_num = self.output_col_idx.len();
236        match &pk_indices {
237            Some(pk_indices) => FunctionalDependencySet::with_key(col_num, pk_indices),
238            None => FunctionalDependencySet::new(col_num),
239        }
240    }
241}
242
243impl CdcScan {
244    pub fn get_table_columns(&self) -> &[ColumnDesc] {
245        &self.cdc_table_desc.columns
246    }
247
248    /// Get the descs of the output columns.
249    pub fn column_descs(&self) -> Vec<ColumnDesc> {
250        self.output_col_idx
251            .iter()
252            .map(|&i| self.get_table_columns()[i].clone())
253            .collect()
254    }
255}
risingwave_frontend/optimizer/plan_node/generic/cdc_scan.rs

risingwave_frontend/optimizer/plan_node/generic/
cdc_scan.rs
