risingwave_frontend/optimizer/plan_node/
stream_dedup.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use itertools::Itertools;
16use risingwave_common::util::sort_util::OrderType;
17use risingwave_pb::stream_plan::DedupNode;
18use risingwave_pb::stream_plan::stream_node::PbNodeBody;
19
20use super::generic::GenericPlanNode;
21use super::stream::prelude::*;
22use super::utils::{TableCatalogBuilder, impl_distill_by_unit};
23use super::{ExprRewritable, PlanBase, PlanTreeNodeUnary, StreamNode, generic};
24use crate::TableCatalog;
25use crate::optimizer::plan_node::StreamPlanRef as PlanRef;
26use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
27use crate::optimizer::property::StreamKind;
28use crate::stream_fragmenter::BuildFragmentGraphState;
29
30#[derive(Debug, Clone, PartialEq, Eq, Hash)]
31pub struct StreamDedup {
32    pub base: PlanBase<Stream>,
33    core: generic::Dedup<PlanRef>,
34}
35
36impl StreamDedup {
37    pub fn new(core: generic::Dedup<PlanRef>) -> Self {
38        let input = core.input.clone();
39        // A dedup operator must be append-only.
40        assert!(input.append_only());
41
42        let base = PlanBase::new_stream_with_core(
43            &core,
44            input.distribution().clone(),
45            StreamKind::AppendOnly,
46            input.emit_on_window_close(),
47            input.watermark_columns().clone(),
48            input.columns_monotonicity().clone(),
49        );
50        StreamDedup { base, core }
51    }
52
53    /// The dedup columns in the pk order of the state table, and the clean watermark column. The
54    /// first dedup column with a watermark is moved to the front, so that the state is cleaned by a
55    /// pk-prefix watermark.
56    fn state_pk_dedup_cols(&self) -> (Vec<usize>, Option<usize>) {
57        let mut cols = self.core.dedup_cols.clone();
58        let input_watermark_columns = self.core.input.watermark_columns();
59        let pos = cols
60            .iter()
61            .position(|&idx| input_watermark_columns.contains(idx));
62        let clean_col = pos.map(|pos| {
63            let col = cols.remove(pos);
64            cols.insert(0, col);
65            col
66        });
67        (cols, clean_col)
68    }
69
70    pub fn infer_internal_table_catalog(&self) -> TableCatalog {
71        let schema = self.core.schema();
72        let mut builder = TableCatalogBuilder::default();
73
74        schema.fields().iter().for_each(|field| {
75            builder.add_column(field);
76        });
77
78        let (dedup_cols, clean_col) = self.state_pk_dedup_cols();
79        dedup_cols.iter().for_each(|idx| {
80            builder.add_order_column(*idx, OrderType::ascending());
81        });
82        if let Some(clean_col) = clean_col {
83            builder.set_clean_watermark_indices(vec![clean_col]);
84        }
85
86        let read_prefix_len_hint = builder.get_current_pk_len();
87
88        builder.build(
89            self.base.distribution().dist_column_indices().to_vec(),
90            read_prefix_len_hint,
91        )
92    }
93}
94
95// assert!(self.base.append_only());
96impl_distill_by_unit!(StreamDedup, core, "StreamAppendOnlyDedup");
97
98impl PlanTreeNodeUnary<Stream> for StreamDedup {
99    fn input(&self) -> PlanRef {
100        self.core.input.clone()
101    }
102
103    fn clone_with_input(&self, input: PlanRef) -> Self {
104        let mut core = self.core.clone();
105        core.input = input;
106        Self::new(core)
107    }
108}
109
110impl_plan_tree_node_for_unary! { Stream, StreamDedup }
111
112impl StreamNode for StreamDedup {
113    fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody {
114        let table_catalog = self
115            .infer_internal_table_catalog()
116            .with_id(state.gen_table_id_wrapped());
117        PbNodeBody::AppendOnlyDedup(Box::new(DedupNode {
118            state_table: Some(table_catalog.to_internal_table_prost()),
119            dedup_column_indices: self
120                .state_pk_dedup_cols()
121                .0
122                .iter()
123                .map(|idx| *idx as _)
124                .collect_vec(),
125        }))
126    }
127}
128
129impl ExprRewritable<Stream> for StreamDedup {}
130
131impl ExprVisitable for StreamDedup {}
risingwave_frontend/optimizer/plan_node/stream_dedup.rs

risingwave_frontend/optimizer/plan_node/
stream_dedup.rs
