Skip to main content

risingwave_storage/table/batch_table/
mod.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::future::Future;
16use std::ops::Bound::{self, Excluded, Included, Unbounded};
17use std::ops::RangeBounds;
18use std::sync::Arc;
19use std::time::Duration;
20
21use await_tree::{InstrumentAwait, SpanExt};
22use bytes::{Bytes, BytesMut};
23use foyer::Hint;
24use futures::future::try_join_all;
25use futures::{Stream, StreamExt, TryStreamExt};
26use futures_async_stream::try_stream;
27use itertools::Itertools;
28use more_asserts::assert_gt;
29use risingwave_common::array::{ArrayBuilderImpl, ArrayRef, DataChunk};
30use risingwave_common::bitmap::Bitmap;
31use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, TableId, TableOption};
32use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
33use risingwave_common::row::{self, OwnedRow, Row, RowExt};
34use risingwave_common::types::{DataType, ToOwnedDatum};
35use risingwave_common::util::epoch::Epoch;
36use risingwave_common::util::row_serde::*;
37use risingwave_common::util::sort_util::OrderType;
38use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAwareSerde;
39use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, deserialize_datum};
40use risingwave_hummock_sdk::HummockReadEpoch;
41use risingwave_hummock_sdk::key::{
42    CopyFromSlice, TableKeyRange, end_bound_of_prefix, next_key, prefixed_range_with_vnode,
43};
44use risingwave_pb::batch_plan::{PbScanRange, scan_range};
45use risingwave_pb::plan_common::StorageTableDesc;
46use tracing::trace;
47mod vector_index_reader;
48pub use vector_index_reader::VectorIndexReader;
49
50use crate::StateStore;
51use crate::error::{StorageError, StorageResult};
52use crate::hummock::CachePolicy;
53use crate::row_serde::row_serde_util::{serialize_pk, serialize_pk_with_vnode};
54use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
55use crate::row_serde::{ColumnMapping, find_columns_by_ids};
56use crate::store::timeout_auto_rebuild::iter_with_timeout_rebuild;
57use crate::store::{
58    NewReadSnapshotOptions, NextEpochOptions, PrefetchOptions, ReadLogOptions, ReadOptions,
59    StateStoreGet, StateStoreIter, StateStoreIterExt, StateStoreRead, TryWaitEpochOptions,
60};
61use crate::table::merge_sort::NodePeek;
62use crate::table::{
63    ChangeLogRow, KeyedRow, TableDistribution, TableIter, should_calculate_prefix_hint,
64};
65
66pub type PkRangeBounds = (Bound<OwnedRow>, Bound<OwnedRow>);
67
68/// Primary-key scan range decoded from the batch plan scan-range protobuf.
69///
70/// This is shared by batch row scans and snapshot backfill so the protobuf decoding, full-scan
71/// default, and batch range-bound conversion stay in one place.
72#[derive(Clone, Debug)]
73pub struct PkScanRange {
74    /// Equality prefix of the primary key.
75    pub pk_prefix: OwnedRow,
76
77    /// Logical range bounds of the next primary-key column after `pk_prefix`.
78    pub range_bounds: PkRangeBounds,
79}
80
81impl PkScanRange {
82    fn pk_type_at(pk_types: &[DataType], index: usize) -> StorageResult<&DataType> {
83        pk_types.get(index).ok_or_else(|| {
84            StorageError::from(memcomparable::Error::Message(format!(
85                "invalid scan range: primary key index {} exceeds primary key length {}",
86                index,
87                pk_types.len()
88            )))
89        })
90    }
91
92    /// Decode a scan range from the protobuf representation and the primary-key column types.
93    pub fn new(scan_range: PbScanRange, pk_types: Vec<DataType>) -> StorageResult<Self> {
94        let mut index = 0;
95        let pk_prefix = OwnedRow::new(
96            scan_range
97                .eq_conds
98                .iter()
99                .map(|v| -> StorageResult<_> {
100                    let ty = Self::pk_type_at(&pk_types, index)?;
101                    index += 1;
102                    Ok(deserialize_datum(v.as_slice(), ty)?)
103                })
104                .try_collect()?,
105        );
106        if scan_range.lower_bound.is_none() && scan_range.upper_bound.is_none() {
107            return Ok(Self {
108                pk_prefix,
109                ..Self::full()
110            });
111        }
112
113        let build_bound =
114            |bound: &scan_range::Bound, mut index| -> StorageResult<Bound<OwnedRow>> {
115                let range_bound = OwnedRow::new(
116                    bound
117                        .value
118                        .iter()
119                        .map(|v| -> StorageResult<_> {
120                            let ty = Self::pk_type_at(&pk_types, index)?;
121                            index += 1;
122                            Ok(deserialize_datum(v.as_slice(), ty)?)
123                        })
124                        .try_collect()?,
125                );
126                if bound.inclusive {
127                    Ok(Bound::Included(range_bound))
128                } else {
129                    Ok(Bound::Excluded(range_bound))
130                }
131            };
132
133        let range_bounds = match (
134            scan_range.lower_bound.as_ref(),
135            scan_range.upper_bound.as_ref(),
136        ) {
137            (Some(lb), Some(ub)) => (build_bound(lb, index)?, build_bound(ub, index)?),
138            (None, Some(ub)) => (Bound::Unbounded, build_bound(ub, index)?),
139            (Some(lb), None) => (build_bound(lb, index)?, Bound::Unbounded),
140            (None, None) => unreachable!(),
141        };
142        Ok(Self {
143            pk_prefix,
144            range_bounds,
145        })
146    }
147
148    /// Create a scan range for full table scan.
149    pub fn full() -> Self {
150        Self {
151            pk_prefix: OwnedRow::default(),
152            range_bounds: (Bound::Unbounded, Bound::Unbounded),
153        }
154    }
155
156    pub fn build_from_protobuf(
157        scan_ranges: &[PbScanRange],
158        table_desc: &StorageTableDesc,
159    ) -> StorageResult<Vec<Self>> {
160        if scan_ranges.is_empty() {
161            Ok(vec![Self::full()])
162        } else {
163            scan_ranges
164                .iter()
165                .map(|scan_range| Self::from_protobuf(scan_range, table_desc))
166                .try_collect()
167        }
168    }
169
170    pub fn from_protobuf(
171        scan_range: &PbScanRange,
172        table_desc: &StorageTableDesc,
173    ) -> StorageResult<Self> {
174        let pk_types = table_desc
175            .pk
176            .iter()
177            .map(|order| {
178                DataType::from(
179                    table_desc.columns[order.column_index as usize]
180                        .column_type
181                        .as_ref()
182                        .unwrap(),
183                )
184            })
185            .collect_vec();
186        Self::new(scan_range.clone(), pk_types)
187    }
188
189    pub fn convert_to_range_bounds<S: StateStore, SD: ValueRowSerde>(
190        self,
191        table: &BatchTableInner<S, SD>,
192    ) -> PkRangeBounds {
193        let PkScanRange {
194            pk_prefix,
195            range_bounds,
196        } = self;
197
198        // The len of a valid pk_prefix should be less than or equal pk's num.
199        let Some(order_type) = table.pk_serializer().get_order_types().get(pk_prefix.len()) else {
200            return range_bounds;
201        };
202        let (start_bound, end_bound) = if order_type.is_ascending() {
203            (range_bounds.0, range_bounds.1)
204        } else {
205            (range_bounds.1, range_bounds.0)
206        };
207
208        let start_bound_is_bounded = !matches!(start_bound, Bound::Unbounded);
209        let end_bound_is_bounded = !matches!(end_bound, Bound::Unbounded);
210
211        let build_bound = |other_bound_is_bounded: bool, bound, order_type_nulls| match bound {
212            Bound::Unbounded => {
213                if other_bound_is_bounded && order_type_nulls {
214                    // `NULL`s are at the start bound side, we should exclude them to meet SQL
215                    // semantics.
216                    Bound::Excluded(OwnedRow::new(vec![None]))
217                } else {
218                    // Both start and end are unbounded, so we need to select all rows.
219                    Bound::Unbounded
220                }
221            }
222            Bound::Included(x) => Bound::Included(x),
223            Bound::Excluded(x) => Bound::Excluded(x),
224        };
225        let start_bound = build_bound(
226            end_bound_is_bounded,
227            start_bound,
228            order_type.nulls_are_first(),
229        );
230        let end_bound = build_bound(
231            start_bound_is_bounded,
232            end_bound,
233            order_type.nulls_are_last(),
234        );
235        (start_bound, end_bound)
236    }
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    #[test]
244    fn test_pk_scan_range_rejects_out_of_bounds_pk_index() {
245        let scan_range = PbScanRange {
246            eq_conds: vec![vec![]],
247            ..Default::default()
248        };
249        assert!(PkScanRange::new(scan_range, vec![]).is_err());
250
251        let scan_range = PbScanRange {
252            lower_bound: Some(scan_range::Bound {
253                value: vec![vec![]],
254                inclusive: false,
255            }),
256            ..Default::default()
257        };
258        assert!(PkScanRange::new(scan_range, vec![]).is_err());
259    }
260}
261
262/// [`BatchTableInner`] is the interface accessing relational data in KV(`StateStore`) with
263/// row-based encoding format, and is used in batch mode.
264#[derive(Clone)]
265pub struct BatchTableInner<S: StateStore, SD: ValueRowSerde> {
266    /// Id for this table.
267    table_id: TableId,
268
269    /// State store backend.
270    store: S,
271
272    /// The schema of the output columns, i.e., this table VIEWED BY some executor like
273    /// `RowSeqScanExecutor`.
274    schema: Schema,
275
276    /// Used for serializing and deserializing the primary key.
277    pk_serializer: OrderedRowSerde,
278
279    output_indices: Vec<usize>,
280
281    /// the key part of `output_indices`.
282    key_output_indices: Option<Vec<usize>>,
283
284    /// the value part of `output_indices`.
285    value_output_indices: Vec<usize>,
286
287    /// used for deserializing key part of output row from pk.
288    output_row_in_key_indices: Vec<usize>,
289
290    /// Mapping from column id to column index for deserializing the row.
291    mapping: Arc<ColumnMapping>,
292
293    /// The index of system column `_rw_timestamp` in the output columns.
294    epoch_idx: Option<usize>,
295
296    /// Row deserializer to deserialize the value in storage to a row.
297    /// The row can be either complete or partial, depending on whether the row encoding is versioned.
298    row_serde: Arc<SD>,
299
300    /// Indices of primary key.
301    /// Note that the index is based on the all columns of the table, instead of the output ones.
302    // FIXME: revisit constructions and usages.
303    pk_indices: Vec<usize>,
304
305    distribution: TableDistribution,
306
307    /// Used for catalog `table_properties`
308    table_option: TableOption,
309
310    read_prefix_len_hint: usize,
311}
312
313/// `BatchTable` will use [`EitherSerde`] as default so that we can support both versioned and
314/// non-versioned tables with the same type.
315pub type BatchTable<S> = BatchTableInner<S, EitherSerde>;
316
317impl<S: StateStore, SD: ValueRowSerde> std::fmt::Debug for BatchTableInner<S, SD> {
318    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
319        f.debug_struct("BatchTableInner").finish_non_exhaustive()
320    }
321}
322
323// init
324impl<S: StateStore> BatchTableInner<S, EitherSerde> {
325    /// Create a  [`BatchTableInner`] given a complete set of `columns` and a partial
326    /// set of `output_column_ids`.
327    /// When reading from the storage table,
328    /// the chunks or rows will only contain columns with the given ids (`output_column_ids`).
329    /// They will in the same order as the given `output_column_ids`.
330    ///
331    /// NOTE(kwannoel): The `output_column_ids` here may be slightly different
332    /// from those supplied to associated executors.
333    /// These `output_column_ids` may have `pk` appended, since they will be needed to scan from
334    /// storage. The associated executors may not have these `pk` fields.
335    pub fn new_partial(
336        store: S,
337        output_column_ids: Vec<ColumnId>,
338        vnodes: Option<Arc<Bitmap>>,
339        table_desc: &StorageTableDesc,
340    ) -> Self {
341        let table_id = table_desc.table_id;
342        let column_descs = table_desc
343            .columns
344            .iter()
345            .map(ColumnDesc::from)
346            .collect_vec();
347        let order_types: Vec<OrderType> = table_desc
348            .pk
349            .iter()
350            .map(|order| OrderType::from_protobuf(order.get_order_type().unwrap()))
351            .collect();
352
353        let pk_indices = table_desc
354            .pk
355            .iter()
356            .map(|k| k.column_index as usize)
357            .collect_vec();
358
359        let table_option = TableOption {
360            retention_seconds: table_desc.retention_seconds,
361        };
362        let value_indices = table_desc
363            .get_value_indices()
364            .iter()
365            .map(|&k| k as usize)
366            .collect_vec();
367        let prefix_hint_len = table_desc.get_read_prefix_len_hint() as usize;
368        let versioned = table_desc.versioned;
369        let distribution = TableDistribution::new_from_storage_table_desc(vnodes, table_desc);
370
371        Self::new_inner(
372            store,
373            table_id,
374            column_descs,
375            output_column_ids,
376            order_types,
377            pk_indices,
378            distribution,
379            table_option,
380            value_indices,
381            prefix_hint_len,
382            versioned,
383        )
384    }
385
386    pub fn for_test_with_partial_columns(
387        store: S,
388        table_id: TableId,
389        columns: Vec<ColumnDesc>,
390        output_column_ids: Vec<ColumnId>,
391        order_types: Vec<OrderType>,
392        pk_indices: Vec<usize>,
393        value_indices: Vec<usize>,
394    ) -> Self {
395        Self::new_inner(
396            store,
397            table_id,
398            columns,
399            output_column_ids,
400            order_types,
401            pk_indices,
402            TableDistribution::singleton(),
403            Default::default(),
404            value_indices,
405            0,
406            false,
407        )
408    }
409
410    pub fn for_test(
411        store: S,
412        table_id: TableId,
413        columns: Vec<ColumnDesc>,
414        order_types: Vec<OrderType>,
415        pk_indices: Vec<usize>,
416        value_indices: Vec<usize>,
417    ) -> Self {
418        let output_column_ids = columns.iter().map(|c| c.column_id).collect();
419        Self::for_test_with_partial_columns(
420            store,
421            table_id,
422            columns,
423            output_column_ids,
424            order_types,
425            pk_indices,
426            value_indices,
427        )
428    }
429
430    #[allow(clippy::too_many_arguments)]
431    fn new_inner(
432        store: S,
433        table_id: TableId,
434        table_columns: Vec<ColumnDesc>,
435        output_column_ids: Vec<ColumnId>,
436        order_types: Vec<OrderType>,
437        pk_indices: Vec<usize>,
438        distribution: TableDistribution,
439        table_option: TableOption,
440        value_indices: Vec<usize>,
441        read_prefix_len_hint: usize,
442        versioned: bool,
443    ) -> Self {
444        assert_eq!(order_types.len(), pk_indices.len());
445
446        let (output_columns, output_indices) =
447            find_columns_by_ids(&table_columns, &output_column_ids);
448
449        let mut value_output_indices = vec![];
450        let mut key_output_indices = vec![];
451        // system column currently only contains `_rw_timestamp`
452        let mut epoch_idx = None;
453
454        for idx in &output_indices {
455            if value_indices.contains(idx) {
456                value_output_indices.push(*idx);
457            } else if pk_indices.contains(idx) {
458                key_output_indices.push(*idx);
459            } else {
460                assert!(epoch_idx.is_none());
461                epoch_idx = Some(*idx);
462            }
463        }
464
465        let output_row_in_key_indices = key_output_indices
466            .iter()
467            .map(|&di| pk_indices.iter().position(|&pi| di == pi).unwrap())
468            .collect_vec();
469        let schema = Schema::new(output_columns.iter().map(Into::into).collect());
470
471        let pk_data_types = pk_indices
472            .iter()
473            .map(|i| table_columns[*i].data_type.clone())
474            .collect();
475        let pk_serializer = OrderedRowSerde::new(pk_data_types, order_types);
476        let (row_serde, mapping) = {
477            if versioned {
478                let value_output_indices_dedup = value_output_indices
479                    .iter()
480                    .unique()
481                    .copied()
482                    .collect::<Vec<_>>();
483                let output_row_in_value_output_indices_dedup = value_output_indices
484                    .iter()
485                    .map(|&di| {
486                        value_output_indices_dedup
487                            .iter()
488                            .position(|&pi| di == pi)
489                            .unwrap()
490                    })
491                    .collect_vec();
492                let mapping = ColumnMapping::new(output_row_in_value_output_indices_dedup);
493                let serde =
494                    ColumnAwareSerde::new(value_output_indices_dedup.into(), table_columns.into());
495                (serde.into(), mapping)
496            } else {
497                let output_row_in_value_indices = value_output_indices
498                    .iter()
499                    .map(|&di| value_indices.iter().position(|&pi| di == pi).unwrap())
500                    .collect_vec();
501                let mapping = ColumnMapping::new(output_row_in_value_indices);
502                let serde = BasicSerde::new(value_indices.into(), table_columns.into());
503                (serde.into(), mapping)
504            }
505        };
506
507        let key_output_indices = match key_output_indices.is_empty() {
508            true => None,
509            false => Some(key_output_indices),
510        };
511        Self {
512            table_id,
513            store,
514            schema,
515            pk_serializer,
516            output_indices,
517            key_output_indices,
518            value_output_indices,
519            output_row_in_key_indices,
520            mapping: Arc::new(mapping),
521            epoch_idx,
522            row_serde: Arc::new(row_serde),
523            pk_indices,
524            distribution,
525            table_option,
526            read_prefix_len_hint,
527        }
528    }
529}
530
531impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
532    pub fn pk_serializer(&self) -> &OrderedRowSerde {
533        &self.pk_serializer
534    }
535
536    pub fn schema(&self) -> &Schema {
537        &self.schema
538    }
539
540    pub fn pk_indices(&self) -> &[usize] {
541        &self.pk_indices
542    }
543
544    pub fn output_indices(&self) -> &[usize] {
545        &self.output_indices
546    }
547
548    /// Get the indices of the primary key columns in the output columns.
549    ///
550    /// Returns `None` if any of the primary key columns is not in the output columns.
551    pub fn pk_in_output_indices(&self) -> Option<Vec<usize>> {
552        self.pk_indices
553            .iter()
554            .map(|&i| self.output_indices.iter().position(|&j| i == j))
555            .collect()
556    }
557
558    pub fn table_id(&self) -> TableId {
559        self.table_id
560    }
561
562    pub fn vnodes(&self) -> &Arc<Bitmap> {
563        self.distribution.vnodes()
564    }
565}
566/// Point get
567impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
568    /// Get a single row by point get
569    pub async fn get_row(
570        &self,
571        pk: impl Row,
572        wait_epoch: HummockReadEpoch,
573    ) -> StorageResult<Option<OwnedRow>> {
574        self.store
575            .try_wait_epoch(
576                wait_epoch,
577                TryWaitEpochOptions {
578                    table_id: self.table_id,
579                },
580            )
581            .await?;
582        let serialized_pk = serialize_pk_with_vnode(
583            &pk,
584            &self.pk_serializer,
585            self.distribution.compute_vnode_by_pk(&pk),
586        );
587        assert!(pk.len() <= self.pk_indices.len());
588
589        let prefix_hint =
590            if should_calculate_prefix_hint(self.read_prefix_len_hint, pk.len(), false) {
591                Some(serialized_pk.slice(VirtualNode::SIZE..))
592            } else {
593                None
594            };
595
596        let read_options = ReadOptions {
597            prefix_hint,
598            cache_policy: CachePolicy::Fill(Hint::Normal),
599            ..Default::default()
600        };
601        let read_snapshot = self
602            .store
603            .new_read_snapshot(
604                wait_epoch,
605                NewReadSnapshotOptions {
606                    table_id: self.table_id,
607                    table_option: self.table_option,
608                },
609            )
610            .await?;
611        match read_snapshot
612            .on_key_value(serialized_pk, read_options, move |key, value| {
613                let row = self.row_serde.deserialize(value)?;
614                Ok((key.epoch_with_gap.pure_epoch(), row))
615            })
616            .await?
617        {
618            Some((epoch, row)) => {
619                let result_row_in_value = self.mapping.project(OwnedRow::new(row));
620
621                match &self.key_output_indices {
622                    Some(key_output_indices) => {
623                        let result_row_in_key =
624                            pk.project(&self.output_row_in_key_indices).into_owned_row();
625                        let mut result_row_vec = vec![];
626                        for idx in &self.output_indices {
627                            if let Some(epoch_idx) = self.epoch_idx
628                                && *idx == epoch_idx
629                            {
630                                let epoch = Epoch::from(epoch);
631                                result_row_vec
632                                    .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
633                            } else if self.value_output_indices.contains(idx) {
634                                let item_position_in_value_indices = &self
635                                    .value_output_indices
636                                    .iter()
637                                    .position(|p| idx == p)
638                                    .unwrap();
639                                result_row_vec.push(
640                                    result_row_in_value
641                                        .datum_at(*item_position_in_value_indices)
642                                        .to_owned_datum(),
643                                );
644                            } else {
645                                let item_position_in_pk_indices =
646                                    key_output_indices.iter().position(|p| idx == p).unwrap();
647                                result_row_vec.push(
648                                    result_row_in_key
649                                        .datum_at(item_position_in_pk_indices)
650                                        .to_owned_datum(),
651                                );
652                            }
653                        }
654                        let result_row = OwnedRow::new(result_row_vec);
655                        Ok(Some(result_row))
656                    }
657                    None => match &self.epoch_idx {
658                        Some(epoch_idx) => {
659                            let mut result_row_vec = vec![];
660                            for idx in &self.output_indices {
661                                if idx == epoch_idx {
662                                    let epoch = Epoch::from(epoch);
663                                    result_row_vec.push(risingwave_common::types::Datum::from(
664                                        epoch.as_scalar(),
665                                    ));
666                                } else {
667                                    let item_position_in_value_indices = &self
668                                        .value_output_indices
669                                        .iter()
670                                        .position(|p| idx == p)
671                                        .unwrap();
672                                    result_row_vec.push(
673                                        result_row_in_value
674                                            .datum_at(*item_position_in_value_indices)
675                                            .to_owned_datum(),
676                                    );
677                                }
678                            }
679                            let result_row = OwnedRow::new(result_row_vec);
680                            Ok(Some(result_row))
681                        }
682                        None => Ok(Some(result_row_in_value.into_owned_row())),
683                    },
684                }
685            }
686            _ => Ok(None),
687        }
688    }
689
690    /// Update the vnode bitmap of the storage table, returns the previous vnode bitmap.
691    #[must_use = "the executor should decide whether to manipulate the cache based on the previous vnode bitmap"]
692    pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
693        self.distribution.update_vnode_bitmap(new_vnodes)
694    }
695}
696
697/// The row iterator of the storage table.
698/// The wrapper of stream item `StorageResult<OwnedRow>` if pk is not persisted.
699impl<S: Stream<Item = StorageResult<OwnedRow>> + Send + Unpin> TableIter for S {
700    async fn next_row(&mut self) -> StorageResult<Option<OwnedRow>> {
701        self.next().await.transpose()
702    }
703}
704
705mod merge_vnode_stream {
706
707    use bytes::Bytes;
708    use futures::{Stream, StreamExt, TryStreamExt};
709    use risingwave_hummock_sdk::key::TableKey;
710
711    use crate::error::StorageResult;
712    use crate::table::KeyedRow;
713    use crate::table::merge_sort::{NodePeek, merge_sort};
714
715    pub(super) enum VnodeStreamType<RowSt, KeyedRowSt> {
716        Single(RowSt),
717        Unordered(Vec<RowSt>),
718        Ordered(Vec<KeyedRowSt>),
719    }
720
721    pub(super) type MergedVnodeStream<
722        R: Send,
723        RowSt: Stream<Item = StorageResult<((), R)>> + Send,
724        KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
725    >
726    where
727        KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
728    = impl Stream<Item = StorageResult<R>> + Send;
729
730    pub(super) type SortKeyType = Bytes; // TODO: may use Vec
731
732    #[define_opaque(MergedVnodeStream)]
733    pub(super) fn merge_stream<
734        R: Send,
735        RowSt: Stream<Item = StorageResult<((), R)>> + Send,
736        KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
737    >(
738        stream: VnodeStreamType<RowSt, KeyedRowSt>,
739    ) -> MergedVnodeStream<R, RowSt, KeyedRowSt>
740    where
741        KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
742    {
743        #[auto_enums::auto_enum(futures03::Stream)]
744        match stream {
745            VnodeStreamType::Single(stream) => stream.map_ok(|(_, row)| row),
746            VnodeStreamType::Unordered(streams) => futures::stream::iter(
747                streams
748                    .into_iter()
749                    .map(|stream| Box::pin(stream.map_ok(|(_, row)| row))),
750            )
751            .flatten_unordered(1024),
752            VnodeStreamType::Ordered(streams) => merge_sort(streams.into_iter().map(|stream| {
753                Box::pin(stream.map_ok(|(key, row)| KeyedRow {
754                    vnode_prefixed_key: TableKey(key),
755                    row,
756                }))
757            }))
758            .map_ok(|keyed_row| keyed_row.row),
759        }
760    }
761}
762
763use merge_vnode_stream::*;
764
765async fn build_vnode_stream<
766    R: Send,
767    RowSt: Stream<Item = StorageResult<((), R)>> + Send,
768    KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
769    RowStFut: Future<Output = StorageResult<RowSt>>,
770    KeyedRowStFut: Future<Output = StorageResult<KeyedRowSt>>,
771>(
772    row_stream_fn: impl Fn(VirtualNode) -> RowStFut,
773    keyed_row_stream_fn: impl Fn(VirtualNode) -> KeyedRowStFut,
774    vnodes: &[VirtualNode],
775    ordered: bool,
776) -> StorageResult<MergedVnodeStream<R, RowSt, KeyedRowSt>>
777where
778    KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
779{
780    let stream = match vnodes {
781        [] => unreachable!(),
782        [vnode] => VnodeStreamType::Single(row_stream_fn(*vnode).await?),
783        // Concat all iterators if not to preserve order.
784        vnodes if !ordered => VnodeStreamType::Unordered(
785            try_join_all(vnodes.iter().map(|vnode| row_stream_fn(*vnode))).await?,
786        ),
787        // Merge all iterators if to preserve order.
788        vnodes => VnodeStreamType::Ordered(
789            try_join_all(vnodes.iter().map(|vnode| keyed_row_stream_fn(*vnode))).await?,
790        ),
791    };
792    Ok(merge_stream(stream))
793}
794
795/// Iterators
796impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
797    /// Get multiple stream item `StorageResult<OwnedRow>` based on the specified vnodes of this table with
798    /// `vnode_hint`, and merge or concat them by given `ordered`.
799    async fn iter_with_encoded_key_range(
800        &self,
801        prefix_hint: Option<Bytes>,
802        (start_bound, end_bound): (Bound<Bytes>, Bound<Bytes>),
803        wait_epoch: HummockReadEpoch,
804        vnode_hint: Option<VirtualNode>,
805        ordered: bool,
806        prefetch_options: PrefetchOptions,
807    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
808    {
809        let vnodes = match vnode_hint {
810            // If `vnode_hint` is set, we can only access this single vnode.
811            Some(vnode) => {
812                assert!(
813                    self.distribution.vnodes().is_set(vnode.to_index()),
814                    "vnode unset: {:?}, distribution: {:?}",
815                    vnode,
816                    self.distribution
817                );
818                vec![vnode]
819            }
820            // Otherwise, we need to access all vnodes of this table.
821            None => self.distribution.vnodes().iter_vnodes().collect_vec(),
822        };
823
824        let read_snapshot = self
825            .store
826            .new_read_snapshot(
827                wait_epoch,
828                NewReadSnapshotOptions {
829                    table_id: self.table_id,
830                    table_option: self.table_option,
831                },
832            )
833            .await?;
834
835        build_vnode_stream(
836            |vnode| {
837                self.iter_vnode_with_encoded_key_range(
838                    &read_snapshot,
839                    prefix_hint.clone(),
840                    (start_bound.as_ref(), end_bound.as_ref()),
841                    vnode,
842                    prefetch_options,
843                )
844            },
845            |vnode| {
846                self.iter_vnode_with_encoded_key_range(
847                    &read_snapshot,
848                    prefix_hint.clone(),
849                    (start_bound.as_ref(), end_bound.as_ref()),
850                    vnode,
851                    prefetch_options,
852                )
853            },
854            &vnodes,
855            ordered,
856        )
857        .await
858    }
859
860    async fn iter_vnode_with_encoded_key_range<K: CopyFromSlice>(
861        &self,
862        read_snapshot: &S::ReadSnapshot,
863        prefix_hint: Option<Bytes>,
864        encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
865        vnode: VirtualNode,
866        prefetch_options: PrefetchOptions,
867    ) -> StorageResult<impl Stream<Item = StorageResult<(K, OwnedRow)>> + Send + use<K, S, SD>>
868    {
869        let (table_key_range, read_options, pk_serializer) =
870            self.vnode_read_context(prefix_hint, encoded_key_range, vnode, prefetch_options);
871
872        let iter = read_snapshot.iter(table_key_range, read_options).await?;
873        Ok(self.iter_stream_from_state_store_iter::<K, _>(iter, pk_serializer))
874    }
875
876    fn vnode_read_context(
877        &self,
878        prefix_hint: Option<Bytes>,
879        encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
880        vnode: VirtualNode,
881        prefetch_options: PrefetchOptions,
882    ) -> (TableKeyRange, ReadOptions, Option<Arc<OrderedRowSerde>>) {
883        let cache_policy = match &encoded_key_range {
884            // To prevent unbounded range scan queries from polluting the block cache, use the
885            // low priority fill policy.
886            (Unbounded, _) | (_, Unbounded) => CachePolicy::Fill(Hint::Low),
887            _ => CachePolicy::Fill(Hint::Normal),
888        };
889
890        let table_key_range = prefixed_range_with_vnode::<&Bytes>(encoded_key_range, vnode);
891        {
892            {
893                let read_options = ReadOptions {
894                    prefix_hint,
895                    prefetch_options,
896                    cache_policy,
897                };
898                let pk_serializer = match self.output_row_in_key_indices.is_empty() {
899                    true => None,
900                    false => Some(Arc::new(self.pk_serializer.clone())),
901                };
902
903                (table_key_range, read_options, pk_serializer)
904            }
905        }
906    }
907
908    fn iter_stream_from_state_store_iter<K: CopyFromSlice, SI: StateStoreIter + Send>(
909        &self,
910        iter: SI,
911        pk_serializer: Option<Arc<OrderedRowSerde>>,
912    ) -> impl Stream<Item = StorageResult<(K, OwnedRow)>> + Send + use<K, SI, S, SD> {
913        BatchTableInnerIterInner {
914            iter,
915            mapping: self.mapping.clone(),
916            epoch_idx: self.epoch_idx,
917            row_deserializer: self.row_serde.clone(),
918            pk_serializer,
919            output_indices: self.output_indices.clone(),
920            key_output_indices: self.key_output_indices.clone(),
921            value_output_indices: self.value_output_indices.clone(),
922            output_row_in_key_indices: self.output_row_in_key_indices.clone(),
923        }
924        .into_stream::<K>()
925    }
926
927    /// Serializes a pk range. A start bound that excludes the largest key leaves nothing to scan,
928    /// which becomes the empty range `[key, key)`.
929    fn serialize_pk_bounds(
930        &self,
931        pk_prefix: impl Row,
932        range_bounds: &impl RangeBounds<OwnedRow>,
933    ) -> (Bound<Bytes>, Bound<Bytes>) {
934        let start_key = self.serialize_pk_bound(&pk_prefix, range_bounds.start_bound(), true);
935        let end_key = self.serialize_pk_bound(&pk_prefix, range_bounds.end_bound(), false);
936        match start_key {
937            Excluded(key) => (Included(key.clone()), Excluded(key)),
938            start_key => (start_key, end_key),
939        }
940    }
941
942    // TODO: directly use `prefixed_range`.
943    /// A start bound is only `Excluded` when no key comes after it. See [`Self::serialize_pk_bounds`].
944    fn serialize_pk_bound(
945        &self,
946        pk_prefix: impl Row,
947        range_bound: Bound<&OwnedRow>,
948        is_start_bound: bool,
949    ) -> Bound<Bytes> {
950        match range_bound {
951            Included(k) => {
952                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
953                let key = pk_prefix.chain(k);
954                let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
955                if is_start_bound {
956                    Included(serialized_key)
957                } else {
958                    // Should use excluded next key for end bound.
959                    // Otherwise keys starting with the bound is not included.
960                    end_bound_of_prefix(&serialized_key)
961                }
962            }
963            Excluded(k) => {
964                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
965                let key = pk_prefix.chain(k);
966                let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
967                if is_start_bound {
968                    // Storage doesn't support excluded begin key yet, so transform it to
969                    // included. There is no next key if a descending key encodes as all 0xff.
970                    let next_serialized_key = next_key(&serialized_key);
971                    if next_serialized_key.is_empty() {
972                        Excluded(serialized_key)
973                    } else {
974                        Included(Bytes::from(next_serialized_key))
975                    }
976                } else {
977                    Excluded(serialized_key)
978                }
979            }
980            Unbounded => {
981                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len());
982                let serialized_pk_prefix = serialize_pk(&pk_prefix, &pk_prefix_serializer);
983                if pk_prefix.is_empty() {
984                    Unbounded
985                } else if is_start_bound {
986                    Included(serialized_pk_prefix)
987                } else {
988                    end_bound_of_prefix(&serialized_pk_prefix)
989                }
990            }
991        }
992    }
993
994    /// Iterates on the table with the given prefix of the pk in `pk_prefix` and the range bounds.
995    async fn iter_with_pk_bounds(
996        &self,
997        epoch: HummockReadEpoch,
998        pk_prefix: impl Row,
999        range_bounds: impl RangeBounds<OwnedRow>,
1000        ordered: bool,
1001        prefetch_options: PrefetchOptions,
1002    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1003        let (start_key, end_key) = self.serialize_pk_bounds(&pk_prefix, &range_bounds);
1004        assert!(pk_prefix.len() <= self.pk_indices.len());
1005        let pk_prefix_indices = (0..pk_prefix.len())
1006            .map(|index| self.pk_indices[index])
1007            .collect_vec();
1008
1009        let prefix_hint = if should_calculate_prefix_hint(
1010            self.read_prefix_len_hint,
1011            pk_prefix.len(),
1012            true,
1013        ) {
1014            let encoded_prefix = if let Bound::Included(start_key) = start_key.as_ref() {
1015                start_key
1016            } else {
1017                unreachable!()
1018            };
1019            let prefix_len = self
1020                .pk_serializer
1021                .deserialize_prefix_len(encoded_prefix, self.read_prefix_len_hint)?;
1022            Some(Bytes::from(encoded_prefix[..prefix_len].to_vec()))
1023        } else {
1024            trace!(
1025                "iter_with_pk_bounds dist_key_indices table_id {} not match prefix pk_prefix {:?}  pk_prefix_indices {:?}",
1026                self.table_id, pk_prefix, pk_prefix_indices
1027            );
1028            None
1029        };
1030
1031        trace!(
1032            "iter_with_pk_bounds table_id {} prefix_hint {:?} start_key: {:?}, end_key: {:?} pk_prefix {:?}  pk_prefix_indices {:?}",
1033            self.table_id, prefix_hint, start_key, end_key, pk_prefix, pk_prefix_indices
1034        );
1035
1036        self.iter_with_encoded_key_range(
1037            prefix_hint,
1038            (start_key, end_key),
1039            epoch,
1040            self.distribution.try_compute_vnode_by_pk_prefix(pk_prefix),
1041            ordered,
1042            prefetch_options,
1043        )
1044        .await
1045    }
1046
1047    // Construct a stream of (columns, row_count) from a row stream
1048    #[try_stream(ok = (Vec<ArrayRef>, usize), error = StorageError)]
1049    async fn convert_row_stream_to_array_vec_stream(
1050        iter: impl Stream<Item = StorageResult<OwnedRow>>,
1051        schema: Schema,
1052        chunk_size: usize,
1053    ) {
1054        use futures::{TryStreamExt, pin_mut};
1055        use risingwave_common::util::iter_util::ZipEqFast;
1056
1057        pin_mut!(iter);
1058
1059        let mut builders: Option<Vec<ArrayBuilderImpl>> = None;
1060        let mut row_count = 0;
1061
1062        while let Some(row) = iter.try_next().await? {
1063            row_count += 1;
1064            // Uses ArrayBuilderImpl instead of DataChunkBuilder here to demonstrate how to build chunk in a columnar manner
1065            let builders_ref =
1066                builders.get_or_insert_with(|| schema.create_array_builders(chunk_size));
1067            for (datum, builder) in row.iter().zip_eq_fast(builders_ref.iter_mut()) {
1068                builder.append(datum);
1069            }
1070            if row_count == chunk_size {
1071                let columns: Vec<_> = builders
1072                    .take()
1073                    .unwrap()
1074                    .into_iter()
1075                    .map(|builder| builder.finish().into())
1076                    .collect();
1077                yield (columns, row_count);
1078                assert!(builders.is_none());
1079                row_count = 0;
1080            }
1081        }
1082
1083        if let Some(builders) = builders {
1084            assert_gt!(row_count, 0);
1085            // yield the last chunk if any
1086            let columns: Vec<_> = builders
1087                .into_iter()
1088                .map(|builder| builder.finish().into())
1089                .collect();
1090            yield (columns, row_count);
1091        }
1092    }
1093
1094    /// Iterates on the table with the given prefix of the pk in `pk_prefix` and the range bounds.
1095    /// Returns a stream of chunks of columns with the provided `chunk_size`
1096    async fn chunk_iter_with_pk_bounds(
1097        &self,
1098        epoch: HummockReadEpoch,
1099        pk_prefix: impl Row,
1100        range_bounds: impl RangeBounds<OwnedRow>,
1101        ordered: bool,
1102        chunk_size: usize,
1103        prefetch_options: PrefetchOptions,
1104    ) -> StorageResult<impl Stream<Item = StorageResult<(Vec<ArrayRef>, usize)>> + Send> {
1105        let iter = self
1106            .iter_with_pk_bounds(epoch, pk_prefix, range_bounds, ordered, prefetch_options)
1107            .await?;
1108
1109        Ok(Self::convert_row_stream_to_array_vec_stream(
1110            iter,
1111            self.schema.clone(),
1112            chunk_size,
1113        ))
1114    }
1115
1116    /// Construct a stream item `StorageResult<OwnedRow>` for batch executors.
1117    /// Differs from the streaming one, this iterator will wait for the epoch before iteration
1118    pub async fn batch_iter_with_pk_bounds(
1119        &self,
1120        epoch: HummockReadEpoch,
1121        pk_prefix: impl Row,
1122        range_bounds: impl RangeBounds<OwnedRow>,
1123        ordered: bool,
1124        prefetch_options: PrefetchOptions,
1125    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1126        self.iter_with_pk_bounds(epoch, pk_prefix, range_bounds, ordered, prefetch_options)
1127            .await
1128    }
1129
1130    // The returned iterator will iterate data from a snapshot corresponding to the given `epoch`.
1131    pub async fn batch_iter(
1132        &self,
1133        epoch: HummockReadEpoch,
1134        ordered: bool,
1135        prefetch_options: PrefetchOptions,
1136    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1137        self.batch_iter_with_pk_bounds(epoch, row::empty(), .., ordered, prefetch_options)
1138            .await
1139    }
1140
1141    fn start_bound_from_pk(&self, start_pk: Option<&OwnedRow>) -> Bound<Bytes> {
1142        if let Some(start_pk) = start_pk {
1143            let mut bytes = BytesMut::new();
1144            self.pk_serializer.serialize(start_pk, &mut bytes);
1145            let bytes = bytes.freeze();
1146            Included(bytes)
1147        } else {
1148            Unbounded
1149        }
1150    }
1151
1152    pub async fn batch_iter_vnode(
1153        &self,
1154        epoch: HummockReadEpoch,
1155        start_pk: Option<&OwnedRow>,
1156        vnode: VirtualNode,
1157        prefetch_options: PrefetchOptions,
1158        rebuild_interval: Duration,
1159    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
1160    {
1161        let full_pk_prefix = OwnedRow::default();
1162        let full_range = (Bound::Unbounded, Bound::Unbounded);
1163        self.batch_iter_vnode_with_pk_range(
1164            epoch,
1165            start_pk,
1166            &full_pk_prefix,
1167            &full_range,
1168            vnode,
1169            prefetch_options,
1170            rebuild_interval,
1171        )
1172        .await
1173    }
1174
1175    /// Iterates data from the given vnode, optionally restricted by a primary-key
1176    /// equality prefix and range bounds on the next primary-key column.
1177    pub async fn batch_iter_vnode_with_pk_range(
1178        &self,
1179        epoch: HummockReadEpoch,
1180        start_pk: Option<&OwnedRow>,
1181        pk_prefix: &OwnedRow,
1182        range_bounds: &(Bound<OwnedRow>, Bound<OwnedRow>),
1183        vnode: VirtualNode,
1184        prefetch_options: PrefetchOptions,
1185        rebuild_interval: Duration,
1186    ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
1187    {
1188        assert!(
1189            !rebuild_interval.is_zero(),
1190            "rebuild_interval should be positive"
1191        );
1192
1193        let normalized_range_bounds = PkScanRange {
1194            pk_prefix: pk_prefix.clone(),
1195            range_bounds: range_bounds.clone(),
1196        }
1197        .convert_to_range_bounds(self);
1198
1199        let (start_key, end_key) = self.serialize_pk_bounds(pk_prefix, &normalized_range_bounds);
1200        let start_key = match start_pk {
1201            Some(start_pk) => self.start_bound_from_pk(Some(start_pk)),
1202            None => start_key,
1203        };
1204
1205        let prefix_hint =
1206            if self.read_prefix_len_hint != 0 && self.read_prefix_len_hint <= pk_prefix.len() {
1207                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len());
1208                let serialized_pk_prefix = serialize_pk(pk_prefix, &pk_prefix_serializer);
1209                let prefix_len = self
1210                    .pk_serializer
1211                    .deserialize_prefix_len(&serialized_pk_prefix, self.read_prefix_len_hint)?;
1212                Some(Bytes::from(serialized_pk_prefix[..prefix_len].to_vec()))
1213            } else {
1214                None
1215            };
1216
1217        let snapshot = Arc::new(
1218            self.store
1219                .new_read_snapshot(
1220                    epoch,
1221                    NewReadSnapshotOptions {
1222                        table_id: self.table_id,
1223                        table_option: self.table_option,
1224                    },
1225                )
1226                .await?,
1227        );
1228        let (table_key_range, read_options, pk_serializer) = self.vnode_read_context(
1229            prefix_hint,
1230            (start_key.as_ref(), end_key.as_ref()),
1231            vnode,
1232            prefetch_options,
1233        );
1234        let iter = iter_with_timeout_rebuild(
1235            snapshot,
1236            table_key_range,
1237            self.table_id,
1238            read_options,
1239            rebuild_interval,
1240        )
1241        .await?;
1242        let iter = self.iter_stream_from_state_store_iter::<(), _>(iter, pk_serializer);
1243        Ok(iter.map_ok(|(_, row)| row))
1244    }
1245
1246    pub async fn next_epoch(&self, epoch: u64) -> StorageResult<u64> {
1247        self.store
1248            .next_epoch(
1249                epoch,
1250                NextEpochOptions {
1251                    table_id: self.table_id,
1252                },
1253            )
1254            .await
1255    }
1256
1257    pub async fn batch_iter_vnode_log(
1258        &self,
1259        start_epoch: u64,
1260        end_epoch: HummockReadEpoch,
1261        start_pk: Option<&OwnedRow>,
1262        vnode: VirtualNode,
1263    ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send + 'static + use<S, SD>>
1264    {
1265        let start_bound = self.start_bound_from_pk(start_pk);
1266        let stream = self
1267            .batch_iter_log_inner::<()>(
1268                start_epoch,
1269                end_epoch,
1270                (start_bound.as_ref(), Unbounded),
1271                vnode,
1272            )
1273            .await?;
1274        Ok(stream.map_ok(|(_, row)| row))
1275    }
1276
1277    pub async fn batch_iter_log_with_pk_bounds(
1278        &self,
1279        start_epoch: u64,
1280        end_epoch: HummockReadEpoch,
1281        ordered: bool,
1282        range_bounds: impl RangeBounds<OwnedRow>,
1283        pk_prefix: impl Row,
1284    ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send> {
1285        let (start_key, end_key) = self.serialize_pk_bounds(&pk_prefix, &range_bounds);
1286        let vnodes = self.distribution.vnodes().iter_vnodes().collect_vec();
1287        build_vnode_stream(
1288            |vnode| {
1289                self.batch_iter_log_inner(
1290                    start_epoch,
1291                    end_epoch,
1292                    (start_key.as_ref(), end_key.as_ref()),
1293                    vnode,
1294                )
1295            },
1296            |vnode| {
1297                self.batch_iter_log_inner(
1298                    start_epoch,
1299                    end_epoch,
1300                    (start_key.as_ref(), end_key.as_ref()),
1301                    vnode,
1302                )
1303            },
1304            &vnodes,
1305            ordered,
1306        )
1307        .await
1308    }
1309
1310    async fn batch_iter_log_inner<K: CopyFromSlice>(
1311        &self,
1312        start_epoch: u64,
1313        end_epoch: HummockReadEpoch,
1314        encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
1315        vnode: VirtualNode,
1316    ) -> StorageResult<impl Stream<Item = StorageResult<(K, ChangeLogRow)>> + Send + use<K, S, SD>>
1317    {
1318        let table_key_range = prefixed_range_with_vnode::<&Bytes>(encoded_key_range, vnode);
1319        let read_options = ReadLogOptions {
1320            table_id: self.table_id,
1321        };
1322        let iter = BatchTableInnerIterLogInner::<S, SD>::new(
1323            &self.store,
1324            self.mapping.clone(),
1325            self.row_serde.clone(),
1326            table_key_range,
1327            read_options,
1328            start_epoch,
1329            end_epoch,
1330        )
1331        .await?
1332        .into_stream::<K>();
1333
1334        Ok(iter)
1335    }
1336
1337    /// Iterates on the table with the given prefix of the pk in `pk_prefix` and the range bounds.
1338    /// Returns a stream of `DataChunk` with the provided `chunk_size`
1339    pub async fn batch_chunk_iter_with_pk_bounds(
1340        &self,
1341        epoch: HummockReadEpoch,
1342        pk_prefix: impl Row,
1343        range_bounds: impl RangeBounds<OwnedRow>,
1344        ordered: bool,
1345        chunk_size: usize,
1346        prefetch_options: PrefetchOptions,
1347    ) -> StorageResult<impl Stream<Item = StorageResult<DataChunk>> + Send> {
1348        let iter = self
1349            .chunk_iter_with_pk_bounds(
1350                epoch,
1351                pk_prefix,
1352                range_bounds,
1353                ordered,
1354                chunk_size,
1355                prefetch_options,
1356            )
1357            .await?;
1358
1359        Ok(iter.map(|item| {
1360            let (columns, row_count) = item?;
1361            Ok(DataChunk::new(columns, row_count))
1362        }))
1363    }
1364}
1365
1366/// [`BatchTableInnerIterInner`] iterates on the storage table.
1367struct BatchTableInnerIterInner<SI: StateStoreIter, SD: ValueRowSerde> {
1368    /// An iterator that returns raw bytes from storage.
1369    iter: SI,
1370
1371    mapping: Arc<ColumnMapping>,
1372
1373    /// The index of system column `_rw_timestamp` in the output columns.
1374    epoch_idx: Option<usize>,
1375
1376    row_deserializer: Arc<SD>,
1377
1378    /// Used for serializing and deserializing the primary key.
1379    pk_serializer: Option<Arc<OrderedRowSerde>>,
1380
1381    output_indices: Vec<usize>,
1382
1383    /// the key part of `output_indices`.
1384    key_output_indices: Option<Vec<usize>>,
1385
1386    /// the value part of `output_indices`.
1387    value_output_indices: Vec<usize>,
1388
1389    /// used for deserializing key part of output row from pk.
1390    output_row_in_key_indices: Vec<usize>,
1391}
1392
1393impl<SI: StateStoreIter, SD: ValueRowSerde> BatchTableInnerIterInner<SI, SD> {
1394    /// Yield a row with its primary key.
1395    #[try_stream(ok = (K, OwnedRow), error = StorageError)]
1396    async fn into_stream<K: CopyFromSlice>(mut self) {
1397        while let Some((k, v)) = self
1398            .iter
1399            .try_next()
1400            .instrument_await("storage_table_iter_next".verbose())
1401            .await?
1402        {
1403            let (table_key, value, epoch_with_gap) = (k.user_key.table_key, v, k.epoch_with_gap);
1404            let row = self.row_deserializer.deserialize(value)?;
1405            let result_row_in_value = self.mapping.project(OwnedRow::new(row));
1406            let row = match &self.key_output_indices {
1407                Some(key_output_indices) => {
1408                    let result_row_in_key = match self.pk_serializer.clone() {
1409                        Some(pk_serializer) => {
1410                            let pk = pk_serializer.deserialize(table_key.key_part().as_ref())?;
1411
1412                            pk.project(&self.output_row_in_key_indices).into_owned_row()
1413                        }
1414                        None => OwnedRow::empty(),
1415                    };
1416
1417                    let mut result_row_vec = vec![];
1418                    for idx in &self.output_indices {
1419                        if let Some(epoch_idx) = self.epoch_idx
1420                            && *idx == epoch_idx
1421                        {
1422                            let epoch = Epoch::from(epoch_with_gap.pure_epoch());
1423                            result_row_vec
1424                                .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
1425                        } else if self.value_output_indices.contains(idx) {
1426                            let item_position_in_value_indices = &self
1427                                .value_output_indices
1428                                .iter()
1429                                .position(|p| idx == p)
1430                                .unwrap();
1431                            result_row_vec.push(
1432                                result_row_in_value
1433                                    .datum_at(*item_position_in_value_indices)
1434                                    .to_owned_datum(),
1435                            );
1436                        } else {
1437                            let item_position_in_pk_indices =
1438                                key_output_indices.iter().position(|p| idx == p).unwrap();
1439                            result_row_vec.push(
1440                                result_row_in_key
1441                                    .datum_at(item_position_in_pk_indices)
1442                                    .to_owned_datum(),
1443                            );
1444                        }
1445                    }
1446                    OwnedRow::new(result_row_vec)
1447                }
1448                None => match &self.epoch_idx {
1449                    Some(epoch_idx) => {
1450                        let mut result_row_vec = vec![];
1451                        for idx in &self.output_indices {
1452                            if idx == epoch_idx {
1453                                let epoch = Epoch::from(epoch_with_gap.pure_epoch());
1454                                result_row_vec
1455                                    .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
1456                            } else {
1457                                let item_position_in_value_indices = &self
1458                                    .value_output_indices
1459                                    .iter()
1460                                    .position(|p| idx == p)
1461                                    .unwrap();
1462                                result_row_vec.push(
1463                                    result_row_in_value
1464                                        .datum_at(*item_position_in_value_indices)
1465                                        .to_owned_datum(),
1466                                );
1467                            }
1468                        }
1469                        OwnedRow::new(result_row_vec)
1470                    }
1471                    None => result_row_in_value.into_owned_row(),
1472                },
1473            };
1474            yield (K::copy_from_slice(table_key.as_ref()), row);
1475        }
1476    }
1477}
1478
1479/// [`BatchTableInnerIterLogInner`] iterates on the storage table.
1480struct BatchTableInnerIterLogInner<S: StateStore, SD: ValueRowSerde> {
1481    /// An iterator that returns raw bytes from storage.
1482    iter: S::ChangeLogIter,
1483
1484    mapping: Arc<ColumnMapping>,
1485
1486    row_deserializer: Arc<SD>,
1487}
1488
1489impl<S: StateStore, SD: ValueRowSerde> BatchTableInnerIterLogInner<S, SD> {
1490    /// If `wait_epoch` is true, it will wait for the given epoch to be committed before iteration.
1491    #[allow(clippy::too_many_arguments)]
1492    async fn new(
1493        store: &S,
1494        mapping: Arc<ColumnMapping>,
1495        row_deserializer: Arc<SD>,
1496        table_key_range: TableKeyRange,
1497        read_options: ReadLogOptions,
1498        start_epoch: u64,
1499        end_epoch: HummockReadEpoch,
1500    ) -> StorageResult<Self> {
1501        store
1502            .try_wait_epoch(
1503                end_epoch,
1504                TryWaitEpochOptions {
1505                    table_id: read_options.table_id,
1506                },
1507            )
1508            .await?;
1509        let iter = store
1510            .iter_log(
1511                (
1512                    start_epoch,
1513                    end_epoch
1514                        .get_epoch()
1515                        .expect("change-log reads require an explicit epoch"),
1516                ),
1517                table_key_range,
1518                read_options,
1519            )
1520            .await?;
1521        let iter = Self {
1522            iter,
1523            mapping,
1524            row_deserializer,
1525        };
1526        Ok(iter)
1527    }
1528
1529    /// Yield a row with its primary key.
1530    fn into_stream<K: CopyFromSlice>(self) -> impl Stream<Item = StorageResult<(K, ChangeLogRow)>> {
1531        self.iter.into_stream(move |(table_key, value)| {
1532            value
1533                .try_map(|value| {
1534                    let full_row = self.row_deserializer.deserialize(value)?;
1535                    let row = self
1536                        .mapping
1537                        .project(OwnedRow::new(full_row))
1538                        .into_owned_row();
1539                    Ok(row)
1540                })
1541                .map(|row| (K::copy_from_slice(table_key.as_ref()), row))
1542        })
1543    }
1544}