1use std::future::Future;
16use std::ops::Bound::{self, Excluded, Included, Unbounded};
17use std::ops::RangeBounds;
18use std::sync::Arc;
19use std::time::Duration;
20
21use await_tree::{InstrumentAwait, SpanExt};
22use bytes::{Bytes, BytesMut};
23use foyer::Hint;
24use futures::future::try_join_all;
25use futures::{Stream, StreamExt, TryStreamExt};
26use futures_async_stream::try_stream;
27use itertools::Itertools;
28use more_asserts::assert_gt;
29use risingwave_common::array::{ArrayBuilderImpl, ArrayRef, DataChunk};
30use risingwave_common::bitmap::Bitmap;
31use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, TableId, TableOption};
32use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
33use risingwave_common::row::{self, OwnedRow, Row, RowExt};
34use risingwave_common::types::{DataType, ToOwnedDatum};
35use risingwave_common::util::epoch::Epoch;
36use risingwave_common::util::row_serde::*;
37use risingwave_common::util::sort_util::OrderType;
38use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAwareSerde;
39use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, deserialize_datum};
40use risingwave_hummock_sdk::HummockReadEpoch;
41use risingwave_hummock_sdk::key::{
42 CopyFromSlice, TableKeyRange, end_bound_of_prefix, next_key, prefixed_range_with_vnode,
43};
44use risingwave_pb::batch_plan::{PbScanRange, scan_range};
45use risingwave_pb::plan_common::StorageTableDesc;
46use tracing::trace;
47mod vector_index_reader;
48pub use vector_index_reader::VectorIndexReader;
49
50use crate::StateStore;
51use crate::error::{StorageError, StorageResult};
52use crate::hummock::CachePolicy;
53use crate::row_serde::row_serde_util::{serialize_pk, serialize_pk_with_vnode};
54use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
55use crate::row_serde::{ColumnMapping, find_columns_by_ids};
56use crate::store::timeout_auto_rebuild::iter_with_timeout_rebuild;
57use crate::store::{
58 NewReadSnapshotOptions, NextEpochOptions, PrefetchOptions, ReadLogOptions, ReadOptions,
59 StateStoreGet, StateStoreIter, StateStoreIterExt, StateStoreRead, TryWaitEpochOptions,
60};
61use crate::table::merge_sort::NodePeek;
62use crate::table::{
63 ChangeLogRow, KeyedRow, TableDistribution, TableIter, should_calculate_prefix_hint,
64};
65
66pub type PkRangeBounds = (Bound<OwnedRow>, Bound<OwnedRow>);
67
68#[derive(Clone, Debug)]
73pub struct PkScanRange {
74 pub pk_prefix: OwnedRow,
76
77 pub range_bounds: PkRangeBounds,
79}
80
81impl PkScanRange {
82 fn pk_type_at(pk_types: &[DataType], index: usize) -> StorageResult<&DataType> {
83 pk_types.get(index).ok_or_else(|| {
84 StorageError::from(memcomparable::Error::Message(format!(
85 "invalid scan range: primary key index {} exceeds primary key length {}",
86 index,
87 pk_types.len()
88 )))
89 })
90 }
91
92 pub fn new(scan_range: PbScanRange, pk_types: Vec<DataType>) -> StorageResult<Self> {
94 let mut index = 0;
95 let pk_prefix = OwnedRow::new(
96 scan_range
97 .eq_conds
98 .iter()
99 .map(|v| -> StorageResult<_> {
100 let ty = Self::pk_type_at(&pk_types, index)?;
101 index += 1;
102 Ok(deserialize_datum(v.as_slice(), ty)?)
103 })
104 .try_collect()?,
105 );
106 if scan_range.lower_bound.is_none() && scan_range.upper_bound.is_none() {
107 return Ok(Self {
108 pk_prefix,
109 ..Self::full()
110 });
111 }
112
113 let build_bound =
114 |bound: &scan_range::Bound, mut index| -> StorageResult<Bound<OwnedRow>> {
115 let range_bound = OwnedRow::new(
116 bound
117 .value
118 .iter()
119 .map(|v| -> StorageResult<_> {
120 let ty = Self::pk_type_at(&pk_types, index)?;
121 index += 1;
122 Ok(deserialize_datum(v.as_slice(), ty)?)
123 })
124 .try_collect()?,
125 );
126 if bound.inclusive {
127 Ok(Bound::Included(range_bound))
128 } else {
129 Ok(Bound::Excluded(range_bound))
130 }
131 };
132
133 let range_bounds = match (
134 scan_range.lower_bound.as_ref(),
135 scan_range.upper_bound.as_ref(),
136 ) {
137 (Some(lb), Some(ub)) => (build_bound(lb, index)?, build_bound(ub, index)?),
138 (None, Some(ub)) => (Bound::Unbounded, build_bound(ub, index)?),
139 (Some(lb), None) => (build_bound(lb, index)?, Bound::Unbounded),
140 (None, None) => unreachable!(),
141 };
142 Ok(Self {
143 pk_prefix,
144 range_bounds,
145 })
146 }
147
148 pub fn full() -> Self {
150 Self {
151 pk_prefix: OwnedRow::default(),
152 range_bounds: (Bound::Unbounded, Bound::Unbounded),
153 }
154 }
155
156 pub fn build_from_protobuf(
157 scan_ranges: &[PbScanRange],
158 table_desc: &StorageTableDesc,
159 ) -> StorageResult<Vec<Self>> {
160 if scan_ranges.is_empty() {
161 Ok(vec![Self::full()])
162 } else {
163 scan_ranges
164 .iter()
165 .map(|scan_range| Self::from_protobuf(scan_range, table_desc))
166 .try_collect()
167 }
168 }
169
170 pub fn from_protobuf(
171 scan_range: &PbScanRange,
172 table_desc: &StorageTableDesc,
173 ) -> StorageResult<Self> {
174 let pk_types = table_desc
175 .pk
176 .iter()
177 .map(|order| {
178 DataType::from(
179 table_desc.columns[order.column_index as usize]
180 .column_type
181 .as_ref()
182 .unwrap(),
183 )
184 })
185 .collect_vec();
186 Self::new(scan_range.clone(), pk_types)
187 }
188
189 pub fn convert_to_range_bounds<S: StateStore, SD: ValueRowSerde>(
190 self,
191 table: &BatchTableInner<S, SD>,
192 ) -> PkRangeBounds {
193 let PkScanRange {
194 pk_prefix,
195 range_bounds,
196 } = self;
197
198 let Some(order_type) = table.pk_serializer().get_order_types().get(pk_prefix.len()) else {
200 return range_bounds;
201 };
202 let (start_bound, end_bound) = if order_type.is_ascending() {
203 (range_bounds.0, range_bounds.1)
204 } else {
205 (range_bounds.1, range_bounds.0)
206 };
207
208 let start_bound_is_bounded = !matches!(start_bound, Bound::Unbounded);
209 let end_bound_is_bounded = !matches!(end_bound, Bound::Unbounded);
210
211 let build_bound = |other_bound_is_bounded: bool, bound, order_type_nulls| match bound {
212 Bound::Unbounded => {
213 if other_bound_is_bounded && order_type_nulls {
214 Bound::Excluded(OwnedRow::new(vec![None]))
217 } else {
218 Bound::Unbounded
220 }
221 }
222 Bound::Included(x) => Bound::Included(x),
223 Bound::Excluded(x) => Bound::Excluded(x),
224 };
225 let start_bound = build_bound(
226 end_bound_is_bounded,
227 start_bound,
228 order_type.nulls_are_first(),
229 );
230 let end_bound = build_bound(
231 start_bound_is_bounded,
232 end_bound,
233 order_type.nulls_are_last(),
234 );
235 (start_bound, end_bound)
236 }
237}
238
239#[cfg(test)]
240mod tests {
241 use super::*;
242
243 #[test]
244 fn test_pk_scan_range_rejects_out_of_bounds_pk_index() {
245 let scan_range = PbScanRange {
246 eq_conds: vec![vec![]],
247 ..Default::default()
248 };
249 assert!(PkScanRange::new(scan_range, vec![]).is_err());
250
251 let scan_range = PbScanRange {
252 lower_bound: Some(scan_range::Bound {
253 value: vec![vec![]],
254 inclusive: false,
255 }),
256 ..Default::default()
257 };
258 assert!(PkScanRange::new(scan_range, vec![]).is_err());
259 }
260}
261
262#[derive(Clone)]
265pub struct BatchTableInner<S: StateStore, SD: ValueRowSerde> {
266 table_id: TableId,
268
269 store: S,
271
272 schema: Schema,
275
276 pk_serializer: OrderedRowSerde,
278
279 output_indices: Vec<usize>,
280
281 key_output_indices: Option<Vec<usize>>,
283
284 value_output_indices: Vec<usize>,
286
287 output_row_in_key_indices: Vec<usize>,
289
290 mapping: Arc<ColumnMapping>,
292
293 epoch_idx: Option<usize>,
295
296 row_serde: Arc<SD>,
299
300 pk_indices: Vec<usize>,
304
305 distribution: TableDistribution,
306
307 table_option: TableOption,
309
310 read_prefix_len_hint: usize,
311}
312
313pub type BatchTable<S> = BatchTableInner<S, EitherSerde>;
316
317impl<S: StateStore, SD: ValueRowSerde> std::fmt::Debug for BatchTableInner<S, SD> {
318 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
319 f.debug_struct("BatchTableInner").finish_non_exhaustive()
320 }
321}
322
323impl<S: StateStore> BatchTableInner<S, EitherSerde> {
325 pub fn new_partial(
336 store: S,
337 output_column_ids: Vec<ColumnId>,
338 vnodes: Option<Arc<Bitmap>>,
339 table_desc: &StorageTableDesc,
340 ) -> Self {
341 let table_id = table_desc.table_id;
342 let column_descs = table_desc
343 .columns
344 .iter()
345 .map(ColumnDesc::from)
346 .collect_vec();
347 let order_types: Vec<OrderType> = table_desc
348 .pk
349 .iter()
350 .map(|order| OrderType::from_protobuf(order.get_order_type().unwrap()))
351 .collect();
352
353 let pk_indices = table_desc
354 .pk
355 .iter()
356 .map(|k| k.column_index as usize)
357 .collect_vec();
358
359 let table_option = TableOption {
360 retention_seconds: table_desc.retention_seconds,
361 };
362 let value_indices = table_desc
363 .get_value_indices()
364 .iter()
365 .map(|&k| k as usize)
366 .collect_vec();
367 let prefix_hint_len = table_desc.get_read_prefix_len_hint() as usize;
368 let versioned = table_desc.versioned;
369 let distribution = TableDistribution::new_from_storage_table_desc(vnodes, table_desc);
370
371 Self::new_inner(
372 store,
373 table_id,
374 column_descs,
375 output_column_ids,
376 order_types,
377 pk_indices,
378 distribution,
379 table_option,
380 value_indices,
381 prefix_hint_len,
382 versioned,
383 )
384 }
385
386 pub fn for_test_with_partial_columns(
387 store: S,
388 table_id: TableId,
389 columns: Vec<ColumnDesc>,
390 output_column_ids: Vec<ColumnId>,
391 order_types: Vec<OrderType>,
392 pk_indices: Vec<usize>,
393 value_indices: Vec<usize>,
394 ) -> Self {
395 Self::new_inner(
396 store,
397 table_id,
398 columns,
399 output_column_ids,
400 order_types,
401 pk_indices,
402 TableDistribution::singleton(),
403 Default::default(),
404 value_indices,
405 0,
406 false,
407 )
408 }
409
410 pub fn for_test(
411 store: S,
412 table_id: TableId,
413 columns: Vec<ColumnDesc>,
414 order_types: Vec<OrderType>,
415 pk_indices: Vec<usize>,
416 value_indices: Vec<usize>,
417 ) -> Self {
418 let output_column_ids = columns.iter().map(|c| c.column_id).collect();
419 Self::for_test_with_partial_columns(
420 store,
421 table_id,
422 columns,
423 output_column_ids,
424 order_types,
425 pk_indices,
426 value_indices,
427 )
428 }
429
430 #[allow(clippy::too_many_arguments)]
431 fn new_inner(
432 store: S,
433 table_id: TableId,
434 table_columns: Vec<ColumnDesc>,
435 output_column_ids: Vec<ColumnId>,
436 order_types: Vec<OrderType>,
437 pk_indices: Vec<usize>,
438 distribution: TableDistribution,
439 table_option: TableOption,
440 value_indices: Vec<usize>,
441 read_prefix_len_hint: usize,
442 versioned: bool,
443 ) -> Self {
444 assert_eq!(order_types.len(), pk_indices.len());
445
446 let (output_columns, output_indices) =
447 find_columns_by_ids(&table_columns, &output_column_ids);
448
449 let mut value_output_indices = vec![];
450 let mut key_output_indices = vec![];
451 let mut epoch_idx = None;
453
454 for idx in &output_indices {
455 if value_indices.contains(idx) {
456 value_output_indices.push(*idx);
457 } else if pk_indices.contains(idx) {
458 key_output_indices.push(*idx);
459 } else {
460 assert!(epoch_idx.is_none());
461 epoch_idx = Some(*idx);
462 }
463 }
464
465 let output_row_in_key_indices = key_output_indices
466 .iter()
467 .map(|&di| pk_indices.iter().position(|&pi| di == pi).unwrap())
468 .collect_vec();
469 let schema = Schema::new(output_columns.iter().map(Into::into).collect());
470
471 let pk_data_types = pk_indices
472 .iter()
473 .map(|i| table_columns[*i].data_type.clone())
474 .collect();
475 let pk_serializer = OrderedRowSerde::new(pk_data_types, order_types);
476 let (row_serde, mapping) = {
477 if versioned {
478 let value_output_indices_dedup = value_output_indices
479 .iter()
480 .unique()
481 .copied()
482 .collect::<Vec<_>>();
483 let output_row_in_value_output_indices_dedup = value_output_indices
484 .iter()
485 .map(|&di| {
486 value_output_indices_dedup
487 .iter()
488 .position(|&pi| di == pi)
489 .unwrap()
490 })
491 .collect_vec();
492 let mapping = ColumnMapping::new(output_row_in_value_output_indices_dedup);
493 let serde =
494 ColumnAwareSerde::new(value_output_indices_dedup.into(), table_columns.into());
495 (serde.into(), mapping)
496 } else {
497 let output_row_in_value_indices = value_output_indices
498 .iter()
499 .map(|&di| value_indices.iter().position(|&pi| di == pi).unwrap())
500 .collect_vec();
501 let mapping = ColumnMapping::new(output_row_in_value_indices);
502 let serde = BasicSerde::new(value_indices.into(), table_columns.into());
503 (serde.into(), mapping)
504 }
505 };
506
507 let key_output_indices = match key_output_indices.is_empty() {
508 true => None,
509 false => Some(key_output_indices),
510 };
511 Self {
512 table_id,
513 store,
514 schema,
515 pk_serializer,
516 output_indices,
517 key_output_indices,
518 value_output_indices,
519 output_row_in_key_indices,
520 mapping: Arc::new(mapping),
521 epoch_idx,
522 row_serde: Arc::new(row_serde),
523 pk_indices,
524 distribution,
525 table_option,
526 read_prefix_len_hint,
527 }
528 }
529}
530
531impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
532 pub fn pk_serializer(&self) -> &OrderedRowSerde {
533 &self.pk_serializer
534 }
535
536 pub fn schema(&self) -> &Schema {
537 &self.schema
538 }
539
540 pub fn pk_indices(&self) -> &[usize] {
541 &self.pk_indices
542 }
543
544 pub fn output_indices(&self) -> &[usize] {
545 &self.output_indices
546 }
547
548 pub fn pk_in_output_indices(&self) -> Option<Vec<usize>> {
552 self.pk_indices
553 .iter()
554 .map(|&i| self.output_indices.iter().position(|&j| i == j))
555 .collect()
556 }
557
558 pub fn table_id(&self) -> TableId {
559 self.table_id
560 }
561
562 pub fn vnodes(&self) -> &Arc<Bitmap> {
563 self.distribution.vnodes()
564 }
565}
566impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
568 pub async fn get_row(
570 &self,
571 pk: impl Row,
572 wait_epoch: HummockReadEpoch,
573 ) -> StorageResult<Option<OwnedRow>> {
574 self.store
575 .try_wait_epoch(
576 wait_epoch,
577 TryWaitEpochOptions {
578 table_id: self.table_id,
579 },
580 )
581 .await?;
582 let serialized_pk = serialize_pk_with_vnode(
583 &pk,
584 &self.pk_serializer,
585 self.distribution.compute_vnode_by_pk(&pk),
586 );
587 assert!(pk.len() <= self.pk_indices.len());
588
589 let prefix_hint =
590 if should_calculate_prefix_hint(self.read_prefix_len_hint, pk.len(), false) {
591 Some(serialized_pk.slice(VirtualNode::SIZE..))
592 } else {
593 None
594 };
595
596 let read_options = ReadOptions {
597 prefix_hint,
598 cache_policy: CachePolicy::Fill(Hint::Normal),
599 ..Default::default()
600 };
601 let read_snapshot = self
602 .store
603 .new_read_snapshot(
604 wait_epoch,
605 NewReadSnapshotOptions {
606 table_id: self.table_id,
607 table_option: self.table_option,
608 },
609 )
610 .await?;
611 match read_snapshot
612 .on_key_value(serialized_pk, read_options, move |key, value| {
613 let row = self.row_serde.deserialize(value)?;
614 Ok((key.epoch_with_gap.pure_epoch(), row))
615 })
616 .await?
617 {
618 Some((epoch, row)) => {
619 let result_row_in_value = self.mapping.project(OwnedRow::new(row));
620
621 match &self.key_output_indices {
622 Some(key_output_indices) => {
623 let result_row_in_key =
624 pk.project(&self.output_row_in_key_indices).into_owned_row();
625 let mut result_row_vec = vec![];
626 for idx in &self.output_indices {
627 if let Some(epoch_idx) = self.epoch_idx
628 && *idx == epoch_idx
629 {
630 let epoch = Epoch::from(epoch);
631 result_row_vec
632 .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
633 } else if self.value_output_indices.contains(idx) {
634 let item_position_in_value_indices = &self
635 .value_output_indices
636 .iter()
637 .position(|p| idx == p)
638 .unwrap();
639 result_row_vec.push(
640 result_row_in_value
641 .datum_at(*item_position_in_value_indices)
642 .to_owned_datum(),
643 );
644 } else {
645 let item_position_in_pk_indices =
646 key_output_indices.iter().position(|p| idx == p).unwrap();
647 result_row_vec.push(
648 result_row_in_key
649 .datum_at(item_position_in_pk_indices)
650 .to_owned_datum(),
651 );
652 }
653 }
654 let result_row = OwnedRow::new(result_row_vec);
655 Ok(Some(result_row))
656 }
657 None => match &self.epoch_idx {
658 Some(epoch_idx) => {
659 let mut result_row_vec = vec![];
660 for idx in &self.output_indices {
661 if idx == epoch_idx {
662 let epoch = Epoch::from(epoch);
663 result_row_vec.push(risingwave_common::types::Datum::from(
664 epoch.as_scalar(),
665 ));
666 } else {
667 let item_position_in_value_indices = &self
668 .value_output_indices
669 .iter()
670 .position(|p| idx == p)
671 .unwrap();
672 result_row_vec.push(
673 result_row_in_value
674 .datum_at(*item_position_in_value_indices)
675 .to_owned_datum(),
676 );
677 }
678 }
679 let result_row = OwnedRow::new(result_row_vec);
680 Ok(Some(result_row))
681 }
682 None => Ok(Some(result_row_in_value.into_owned_row())),
683 },
684 }
685 }
686 _ => Ok(None),
687 }
688 }
689
690 #[must_use = "the executor should decide whether to manipulate the cache based on the previous vnode bitmap"]
692 pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
693 self.distribution.update_vnode_bitmap(new_vnodes)
694 }
695}
696
697impl<S: Stream<Item = StorageResult<OwnedRow>> + Send + Unpin> TableIter for S {
700 async fn next_row(&mut self) -> StorageResult<Option<OwnedRow>> {
701 self.next().await.transpose()
702 }
703}
704
705mod merge_vnode_stream {
706
707 use bytes::Bytes;
708 use futures::{Stream, StreamExt, TryStreamExt};
709 use risingwave_hummock_sdk::key::TableKey;
710
711 use crate::error::StorageResult;
712 use crate::table::KeyedRow;
713 use crate::table::merge_sort::{NodePeek, merge_sort};
714
715 pub(super) enum VnodeStreamType<RowSt, KeyedRowSt> {
716 Single(RowSt),
717 Unordered(Vec<RowSt>),
718 Ordered(Vec<KeyedRowSt>),
719 }
720
721 pub(super) type MergedVnodeStream<
722 R: Send,
723 RowSt: Stream<Item = StorageResult<((), R)>> + Send,
724 KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
725 >
726 where
727 KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
728 = impl Stream<Item = StorageResult<R>> + Send;
729
730 pub(super) type SortKeyType = Bytes; #[define_opaque(MergedVnodeStream)]
733 pub(super) fn merge_stream<
734 R: Send,
735 RowSt: Stream<Item = StorageResult<((), R)>> + Send,
736 KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
737 >(
738 stream: VnodeStreamType<RowSt, KeyedRowSt>,
739 ) -> MergedVnodeStream<R, RowSt, KeyedRowSt>
740 where
741 KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
742 {
743 #[auto_enums::auto_enum(futures03::Stream)]
744 match stream {
745 VnodeStreamType::Single(stream) => stream.map_ok(|(_, row)| row),
746 VnodeStreamType::Unordered(streams) => futures::stream::iter(
747 streams
748 .into_iter()
749 .map(|stream| Box::pin(stream.map_ok(|(_, row)| row))),
750 )
751 .flatten_unordered(1024),
752 VnodeStreamType::Ordered(streams) => merge_sort(streams.into_iter().map(|stream| {
753 Box::pin(stream.map_ok(|(key, row)| KeyedRow {
754 vnode_prefixed_key: TableKey(key),
755 row,
756 }))
757 }))
758 .map_ok(|keyed_row| keyed_row.row),
759 }
760 }
761}
762
763use merge_vnode_stream::*;
764
765async fn build_vnode_stream<
766 R: Send,
767 RowSt: Stream<Item = StorageResult<((), R)>> + Send,
768 KeyedRowSt: Stream<Item = StorageResult<(SortKeyType, R)>> + Send,
769 RowStFut: Future<Output = StorageResult<RowSt>>,
770 KeyedRowStFut: Future<Output = StorageResult<KeyedRowSt>>,
771>(
772 row_stream_fn: impl Fn(VirtualNode) -> RowStFut,
773 keyed_row_stream_fn: impl Fn(VirtualNode) -> KeyedRowStFut,
774 vnodes: &[VirtualNode],
775 ordered: bool,
776) -> StorageResult<MergedVnodeStream<R, RowSt, KeyedRowSt>>
777where
778 KeyedRow<SortKeyType, R>: NodePeek + Send + Sync,
779{
780 let stream = match vnodes {
781 [] => unreachable!(),
782 [vnode] => VnodeStreamType::Single(row_stream_fn(*vnode).await?),
783 vnodes if !ordered => VnodeStreamType::Unordered(
785 try_join_all(vnodes.iter().map(|vnode| row_stream_fn(*vnode))).await?,
786 ),
787 vnodes => VnodeStreamType::Ordered(
789 try_join_all(vnodes.iter().map(|vnode| keyed_row_stream_fn(*vnode))).await?,
790 ),
791 };
792 Ok(merge_stream(stream))
793}
794
795impl<S: StateStore, SD: ValueRowSerde> BatchTableInner<S, SD> {
797 async fn iter_with_encoded_key_range(
800 &self,
801 prefix_hint: Option<Bytes>,
802 (start_bound, end_bound): (Bound<Bytes>, Bound<Bytes>),
803 wait_epoch: HummockReadEpoch,
804 vnode_hint: Option<VirtualNode>,
805 ordered: bool,
806 prefetch_options: PrefetchOptions,
807 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
808 {
809 let vnodes = match vnode_hint {
810 Some(vnode) => {
812 assert!(
813 self.distribution.vnodes().is_set(vnode.to_index()),
814 "vnode unset: {:?}, distribution: {:?}",
815 vnode,
816 self.distribution
817 );
818 vec![vnode]
819 }
820 None => self.distribution.vnodes().iter_vnodes().collect_vec(),
822 };
823
824 let read_snapshot = self
825 .store
826 .new_read_snapshot(
827 wait_epoch,
828 NewReadSnapshotOptions {
829 table_id: self.table_id,
830 table_option: self.table_option,
831 },
832 )
833 .await?;
834
835 build_vnode_stream(
836 |vnode| {
837 self.iter_vnode_with_encoded_key_range(
838 &read_snapshot,
839 prefix_hint.clone(),
840 (start_bound.as_ref(), end_bound.as_ref()),
841 vnode,
842 prefetch_options,
843 )
844 },
845 |vnode| {
846 self.iter_vnode_with_encoded_key_range(
847 &read_snapshot,
848 prefix_hint.clone(),
849 (start_bound.as_ref(), end_bound.as_ref()),
850 vnode,
851 prefetch_options,
852 )
853 },
854 &vnodes,
855 ordered,
856 )
857 .await
858 }
859
860 async fn iter_vnode_with_encoded_key_range<K: CopyFromSlice>(
861 &self,
862 read_snapshot: &S::ReadSnapshot,
863 prefix_hint: Option<Bytes>,
864 encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
865 vnode: VirtualNode,
866 prefetch_options: PrefetchOptions,
867 ) -> StorageResult<impl Stream<Item = StorageResult<(K, OwnedRow)>> + Send + use<K, S, SD>>
868 {
869 let (table_key_range, read_options, pk_serializer) =
870 self.vnode_read_context(prefix_hint, encoded_key_range, vnode, prefetch_options);
871
872 let iter = read_snapshot.iter(table_key_range, read_options).await?;
873 Ok(self.iter_stream_from_state_store_iter::<K, _>(iter, pk_serializer))
874 }
875
876 fn vnode_read_context(
877 &self,
878 prefix_hint: Option<Bytes>,
879 encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
880 vnode: VirtualNode,
881 prefetch_options: PrefetchOptions,
882 ) -> (TableKeyRange, ReadOptions, Option<Arc<OrderedRowSerde>>) {
883 let cache_policy = match &encoded_key_range {
884 (Unbounded, _) | (_, Unbounded) => CachePolicy::Fill(Hint::Low),
887 _ => CachePolicy::Fill(Hint::Normal),
888 };
889
890 let table_key_range = prefixed_range_with_vnode::<&Bytes>(encoded_key_range, vnode);
891 {
892 {
893 let read_options = ReadOptions {
894 prefix_hint,
895 prefetch_options,
896 cache_policy,
897 };
898 let pk_serializer = match self.output_row_in_key_indices.is_empty() {
899 true => None,
900 false => Some(Arc::new(self.pk_serializer.clone())),
901 };
902
903 (table_key_range, read_options, pk_serializer)
904 }
905 }
906 }
907
908 fn iter_stream_from_state_store_iter<K: CopyFromSlice, SI: StateStoreIter + Send>(
909 &self,
910 iter: SI,
911 pk_serializer: Option<Arc<OrderedRowSerde>>,
912 ) -> impl Stream<Item = StorageResult<(K, OwnedRow)>> + Send + use<K, SI, S, SD> {
913 BatchTableInnerIterInner {
914 iter,
915 mapping: self.mapping.clone(),
916 epoch_idx: self.epoch_idx,
917 row_deserializer: self.row_serde.clone(),
918 pk_serializer,
919 output_indices: self.output_indices.clone(),
920 key_output_indices: self.key_output_indices.clone(),
921 value_output_indices: self.value_output_indices.clone(),
922 output_row_in_key_indices: self.output_row_in_key_indices.clone(),
923 }
924 .into_stream::<K>()
925 }
926
927 fn serialize_pk_bounds(
930 &self,
931 pk_prefix: impl Row,
932 range_bounds: &impl RangeBounds<OwnedRow>,
933 ) -> (Bound<Bytes>, Bound<Bytes>) {
934 let start_key = self.serialize_pk_bound(&pk_prefix, range_bounds.start_bound(), true);
935 let end_key = self.serialize_pk_bound(&pk_prefix, range_bounds.end_bound(), false);
936 match start_key {
937 Excluded(key) => (Included(key.clone()), Excluded(key)),
938 start_key => (start_key, end_key),
939 }
940 }
941
942 fn serialize_pk_bound(
945 &self,
946 pk_prefix: impl Row,
947 range_bound: Bound<&OwnedRow>,
948 is_start_bound: bool,
949 ) -> Bound<Bytes> {
950 match range_bound {
951 Included(k) => {
952 let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
953 let key = pk_prefix.chain(k);
954 let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
955 if is_start_bound {
956 Included(serialized_key)
957 } else {
958 end_bound_of_prefix(&serialized_key)
961 }
962 }
963 Excluded(k) => {
964 let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
965 let key = pk_prefix.chain(k);
966 let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
967 if is_start_bound {
968 let next_serialized_key = next_key(&serialized_key);
971 if next_serialized_key.is_empty() {
972 Excluded(serialized_key)
973 } else {
974 Included(Bytes::from(next_serialized_key))
975 }
976 } else {
977 Excluded(serialized_key)
978 }
979 }
980 Unbounded => {
981 let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len());
982 let serialized_pk_prefix = serialize_pk(&pk_prefix, &pk_prefix_serializer);
983 if pk_prefix.is_empty() {
984 Unbounded
985 } else if is_start_bound {
986 Included(serialized_pk_prefix)
987 } else {
988 end_bound_of_prefix(&serialized_pk_prefix)
989 }
990 }
991 }
992 }
993
994 async fn iter_with_pk_bounds(
996 &self,
997 epoch: HummockReadEpoch,
998 pk_prefix: impl Row,
999 range_bounds: impl RangeBounds<OwnedRow>,
1000 ordered: bool,
1001 prefetch_options: PrefetchOptions,
1002 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1003 let (start_key, end_key) = self.serialize_pk_bounds(&pk_prefix, &range_bounds);
1004 assert!(pk_prefix.len() <= self.pk_indices.len());
1005 let pk_prefix_indices = (0..pk_prefix.len())
1006 .map(|index| self.pk_indices[index])
1007 .collect_vec();
1008
1009 let prefix_hint = if should_calculate_prefix_hint(
1010 self.read_prefix_len_hint,
1011 pk_prefix.len(),
1012 true,
1013 ) {
1014 let encoded_prefix = if let Bound::Included(start_key) = start_key.as_ref() {
1015 start_key
1016 } else {
1017 unreachable!()
1018 };
1019 let prefix_len = self
1020 .pk_serializer
1021 .deserialize_prefix_len(encoded_prefix, self.read_prefix_len_hint)?;
1022 Some(Bytes::from(encoded_prefix[..prefix_len].to_vec()))
1023 } else {
1024 trace!(
1025 "iter_with_pk_bounds dist_key_indices table_id {} not match prefix pk_prefix {:?} pk_prefix_indices {:?}",
1026 self.table_id, pk_prefix, pk_prefix_indices
1027 );
1028 None
1029 };
1030
1031 trace!(
1032 "iter_with_pk_bounds table_id {} prefix_hint {:?} start_key: {:?}, end_key: {:?} pk_prefix {:?} pk_prefix_indices {:?}",
1033 self.table_id, prefix_hint, start_key, end_key, pk_prefix, pk_prefix_indices
1034 );
1035
1036 self.iter_with_encoded_key_range(
1037 prefix_hint,
1038 (start_key, end_key),
1039 epoch,
1040 self.distribution.try_compute_vnode_by_pk_prefix(pk_prefix),
1041 ordered,
1042 prefetch_options,
1043 )
1044 .await
1045 }
1046
1047 #[try_stream(ok = (Vec<ArrayRef>, usize), error = StorageError)]
1049 async fn convert_row_stream_to_array_vec_stream(
1050 iter: impl Stream<Item = StorageResult<OwnedRow>>,
1051 schema: Schema,
1052 chunk_size: usize,
1053 ) {
1054 use futures::{TryStreamExt, pin_mut};
1055 use risingwave_common::util::iter_util::ZipEqFast;
1056
1057 pin_mut!(iter);
1058
1059 let mut builders: Option<Vec<ArrayBuilderImpl>> = None;
1060 let mut row_count = 0;
1061
1062 while let Some(row) = iter.try_next().await? {
1063 row_count += 1;
1064 let builders_ref =
1066 builders.get_or_insert_with(|| schema.create_array_builders(chunk_size));
1067 for (datum, builder) in row.iter().zip_eq_fast(builders_ref.iter_mut()) {
1068 builder.append(datum);
1069 }
1070 if row_count == chunk_size {
1071 let columns: Vec<_> = builders
1072 .take()
1073 .unwrap()
1074 .into_iter()
1075 .map(|builder| builder.finish().into())
1076 .collect();
1077 yield (columns, row_count);
1078 assert!(builders.is_none());
1079 row_count = 0;
1080 }
1081 }
1082
1083 if let Some(builders) = builders {
1084 assert_gt!(row_count, 0);
1085 let columns: Vec<_> = builders
1087 .into_iter()
1088 .map(|builder| builder.finish().into())
1089 .collect();
1090 yield (columns, row_count);
1091 }
1092 }
1093
1094 async fn chunk_iter_with_pk_bounds(
1097 &self,
1098 epoch: HummockReadEpoch,
1099 pk_prefix: impl Row,
1100 range_bounds: impl RangeBounds<OwnedRow>,
1101 ordered: bool,
1102 chunk_size: usize,
1103 prefetch_options: PrefetchOptions,
1104 ) -> StorageResult<impl Stream<Item = StorageResult<(Vec<ArrayRef>, usize)>> + Send> {
1105 let iter = self
1106 .iter_with_pk_bounds(epoch, pk_prefix, range_bounds, ordered, prefetch_options)
1107 .await?;
1108
1109 Ok(Self::convert_row_stream_to_array_vec_stream(
1110 iter,
1111 self.schema.clone(),
1112 chunk_size,
1113 ))
1114 }
1115
1116 pub async fn batch_iter_with_pk_bounds(
1119 &self,
1120 epoch: HummockReadEpoch,
1121 pk_prefix: impl Row,
1122 range_bounds: impl RangeBounds<OwnedRow>,
1123 ordered: bool,
1124 prefetch_options: PrefetchOptions,
1125 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1126 self.iter_with_pk_bounds(epoch, pk_prefix, range_bounds, ordered, prefetch_options)
1127 .await
1128 }
1129
1130 pub async fn batch_iter(
1132 &self,
1133 epoch: HummockReadEpoch,
1134 ordered: bool,
1135 prefetch_options: PrefetchOptions,
1136 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send> {
1137 self.batch_iter_with_pk_bounds(epoch, row::empty(), .., ordered, prefetch_options)
1138 .await
1139 }
1140
1141 fn start_bound_from_pk(&self, start_pk: Option<&OwnedRow>) -> Bound<Bytes> {
1142 if let Some(start_pk) = start_pk {
1143 let mut bytes = BytesMut::new();
1144 self.pk_serializer.serialize(start_pk, &mut bytes);
1145 let bytes = bytes.freeze();
1146 Included(bytes)
1147 } else {
1148 Unbounded
1149 }
1150 }
1151
1152 pub async fn batch_iter_vnode(
1153 &self,
1154 epoch: HummockReadEpoch,
1155 start_pk: Option<&OwnedRow>,
1156 vnode: VirtualNode,
1157 prefetch_options: PrefetchOptions,
1158 rebuild_interval: Duration,
1159 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
1160 {
1161 let full_pk_prefix = OwnedRow::default();
1162 let full_range = (Bound::Unbounded, Bound::Unbounded);
1163 self.batch_iter_vnode_with_pk_range(
1164 epoch,
1165 start_pk,
1166 &full_pk_prefix,
1167 &full_range,
1168 vnode,
1169 prefetch_options,
1170 rebuild_interval,
1171 )
1172 .await
1173 }
1174
1175 pub async fn batch_iter_vnode_with_pk_range(
1178 &self,
1179 epoch: HummockReadEpoch,
1180 start_pk: Option<&OwnedRow>,
1181 pk_prefix: &OwnedRow,
1182 range_bounds: &(Bound<OwnedRow>, Bound<OwnedRow>),
1183 vnode: VirtualNode,
1184 prefetch_options: PrefetchOptions,
1185 rebuild_interval: Duration,
1186 ) -> StorageResult<impl Stream<Item = StorageResult<OwnedRow>> + Send + 'static + use<S, SD>>
1187 {
1188 assert!(
1189 !rebuild_interval.is_zero(),
1190 "rebuild_interval should be positive"
1191 );
1192
1193 let normalized_range_bounds = PkScanRange {
1194 pk_prefix: pk_prefix.clone(),
1195 range_bounds: range_bounds.clone(),
1196 }
1197 .convert_to_range_bounds(self);
1198
1199 let (start_key, end_key) = self.serialize_pk_bounds(pk_prefix, &normalized_range_bounds);
1200 let start_key = match start_pk {
1201 Some(start_pk) => self.start_bound_from_pk(Some(start_pk)),
1202 None => start_key,
1203 };
1204
1205 let prefix_hint =
1206 if self.read_prefix_len_hint != 0 && self.read_prefix_len_hint <= pk_prefix.len() {
1207 let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len());
1208 let serialized_pk_prefix = serialize_pk(pk_prefix, &pk_prefix_serializer);
1209 let prefix_len = self
1210 .pk_serializer
1211 .deserialize_prefix_len(&serialized_pk_prefix, self.read_prefix_len_hint)?;
1212 Some(Bytes::from(serialized_pk_prefix[..prefix_len].to_vec()))
1213 } else {
1214 None
1215 };
1216
1217 let snapshot = Arc::new(
1218 self.store
1219 .new_read_snapshot(
1220 epoch,
1221 NewReadSnapshotOptions {
1222 table_id: self.table_id,
1223 table_option: self.table_option,
1224 },
1225 )
1226 .await?,
1227 );
1228 let (table_key_range, read_options, pk_serializer) = self.vnode_read_context(
1229 prefix_hint,
1230 (start_key.as_ref(), end_key.as_ref()),
1231 vnode,
1232 prefetch_options,
1233 );
1234 let iter = iter_with_timeout_rebuild(
1235 snapshot,
1236 table_key_range,
1237 self.table_id,
1238 read_options,
1239 rebuild_interval,
1240 )
1241 .await?;
1242 let iter = self.iter_stream_from_state_store_iter::<(), _>(iter, pk_serializer);
1243 Ok(iter.map_ok(|(_, row)| row))
1244 }
1245
1246 pub async fn next_epoch(&self, epoch: u64) -> StorageResult<u64> {
1247 self.store
1248 .next_epoch(
1249 epoch,
1250 NextEpochOptions {
1251 table_id: self.table_id,
1252 },
1253 )
1254 .await
1255 }
1256
1257 pub async fn batch_iter_vnode_log(
1258 &self,
1259 start_epoch: u64,
1260 end_epoch: HummockReadEpoch,
1261 start_pk: Option<&OwnedRow>,
1262 vnode: VirtualNode,
1263 ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send + 'static + use<S, SD>>
1264 {
1265 let start_bound = self.start_bound_from_pk(start_pk);
1266 let stream = self
1267 .batch_iter_log_inner::<()>(
1268 start_epoch,
1269 end_epoch,
1270 (start_bound.as_ref(), Unbounded),
1271 vnode,
1272 )
1273 .await?;
1274 Ok(stream.map_ok(|(_, row)| row))
1275 }
1276
1277 pub async fn batch_iter_log_with_pk_bounds(
1278 &self,
1279 start_epoch: u64,
1280 end_epoch: HummockReadEpoch,
1281 ordered: bool,
1282 range_bounds: impl RangeBounds<OwnedRow>,
1283 pk_prefix: impl Row,
1284 ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send> {
1285 let (start_key, end_key) = self.serialize_pk_bounds(&pk_prefix, &range_bounds);
1286 let vnodes = self.distribution.vnodes().iter_vnodes().collect_vec();
1287 build_vnode_stream(
1288 |vnode| {
1289 self.batch_iter_log_inner(
1290 start_epoch,
1291 end_epoch,
1292 (start_key.as_ref(), end_key.as_ref()),
1293 vnode,
1294 )
1295 },
1296 |vnode| {
1297 self.batch_iter_log_inner(
1298 start_epoch,
1299 end_epoch,
1300 (start_key.as_ref(), end_key.as_ref()),
1301 vnode,
1302 )
1303 },
1304 &vnodes,
1305 ordered,
1306 )
1307 .await
1308 }
1309
1310 async fn batch_iter_log_inner<K: CopyFromSlice>(
1311 &self,
1312 start_epoch: u64,
1313 end_epoch: HummockReadEpoch,
1314 encoded_key_range: (Bound<&Bytes>, Bound<&Bytes>),
1315 vnode: VirtualNode,
1316 ) -> StorageResult<impl Stream<Item = StorageResult<(K, ChangeLogRow)>> + Send + use<K, S, SD>>
1317 {
1318 let table_key_range = prefixed_range_with_vnode::<&Bytes>(encoded_key_range, vnode);
1319 let read_options = ReadLogOptions {
1320 table_id: self.table_id,
1321 };
1322 let iter = BatchTableInnerIterLogInner::<S, SD>::new(
1323 &self.store,
1324 self.mapping.clone(),
1325 self.row_serde.clone(),
1326 table_key_range,
1327 read_options,
1328 start_epoch,
1329 end_epoch,
1330 )
1331 .await?
1332 .into_stream::<K>();
1333
1334 Ok(iter)
1335 }
1336
1337 pub async fn batch_chunk_iter_with_pk_bounds(
1340 &self,
1341 epoch: HummockReadEpoch,
1342 pk_prefix: impl Row,
1343 range_bounds: impl RangeBounds<OwnedRow>,
1344 ordered: bool,
1345 chunk_size: usize,
1346 prefetch_options: PrefetchOptions,
1347 ) -> StorageResult<impl Stream<Item = StorageResult<DataChunk>> + Send> {
1348 let iter = self
1349 .chunk_iter_with_pk_bounds(
1350 epoch,
1351 pk_prefix,
1352 range_bounds,
1353 ordered,
1354 chunk_size,
1355 prefetch_options,
1356 )
1357 .await?;
1358
1359 Ok(iter.map(|item| {
1360 let (columns, row_count) = item?;
1361 Ok(DataChunk::new(columns, row_count))
1362 }))
1363 }
1364}
1365
1366struct BatchTableInnerIterInner<SI: StateStoreIter, SD: ValueRowSerde> {
1368 iter: SI,
1370
1371 mapping: Arc<ColumnMapping>,
1372
1373 epoch_idx: Option<usize>,
1375
1376 row_deserializer: Arc<SD>,
1377
1378 pk_serializer: Option<Arc<OrderedRowSerde>>,
1380
1381 output_indices: Vec<usize>,
1382
1383 key_output_indices: Option<Vec<usize>>,
1385
1386 value_output_indices: Vec<usize>,
1388
1389 output_row_in_key_indices: Vec<usize>,
1391}
1392
1393impl<SI: StateStoreIter, SD: ValueRowSerde> BatchTableInnerIterInner<SI, SD> {
1394 #[try_stream(ok = (K, OwnedRow), error = StorageError)]
1396 async fn into_stream<K: CopyFromSlice>(mut self) {
1397 while let Some((k, v)) = self
1398 .iter
1399 .try_next()
1400 .instrument_await("storage_table_iter_next".verbose())
1401 .await?
1402 {
1403 let (table_key, value, epoch_with_gap) = (k.user_key.table_key, v, k.epoch_with_gap);
1404 let row = self.row_deserializer.deserialize(value)?;
1405 let result_row_in_value = self.mapping.project(OwnedRow::new(row));
1406 let row = match &self.key_output_indices {
1407 Some(key_output_indices) => {
1408 let result_row_in_key = match self.pk_serializer.clone() {
1409 Some(pk_serializer) => {
1410 let pk = pk_serializer.deserialize(table_key.key_part().as_ref())?;
1411
1412 pk.project(&self.output_row_in_key_indices).into_owned_row()
1413 }
1414 None => OwnedRow::empty(),
1415 };
1416
1417 let mut result_row_vec = vec![];
1418 for idx in &self.output_indices {
1419 if let Some(epoch_idx) = self.epoch_idx
1420 && *idx == epoch_idx
1421 {
1422 let epoch = Epoch::from(epoch_with_gap.pure_epoch());
1423 result_row_vec
1424 .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
1425 } else if self.value_output_indices.contains(idx) {
1426 let item_position_in_value_indices = &self
1427 .value_output_indices
1428 .iter()
1429 .position(|p| idx == p)
1430 .unwrap();
1431 result_row_vec.push(
1432 result_row_in_value
1433 .datum_at(*item_position_in_value_indices)
1434 .to_owned_datum(),
1435 );
1436 } else {
1437 let item_position_in_pk_indices =
1438 key_output_indices.iter().position(|p| idx == p).unwrap();
1439 result_row_vec.push(
1440 result_row_in_key
1441 .datum_at(item_position_in_pk_indices)
1442 .to_owned_datum(),
1443 );
1444 }
1445 }
1446 OwnedRow::new(result_row_vec)
1447 }
1448 None => match &self.epoch_idx {
1449 Some(epoch_idx) => {
1450 let mut result_row_vec = vec![];
1451 for idx in &self.output_indices {
1452 if idx == epoch_idx {
1453 let epoch = Epoch::from(epoch_with_gap.pure_epoch());
1454 result_row_vec
1455 .push(risingwave_common::types::Datum::from(epoch.as_scalar()));
1456 } else {
1457 let item_position_in_value_indices = &self
1458 .value_output_indices
1459 .iter()
1460 .position(|p| idx == p)
1461 .unwrap();
1462 result_row_vec.push(
1463 result_row_in_value
1464 .datum_at(*item_position_in_value_indices)
1465 .to_owned_datum(),
1466 );
1467 }
1468 }
1469 OwnedRow::new(result_row_vec)
1470 }
1471 None => result_row_in_value.into_owned_row(),
1472 },
1473 };
1474 yield (K::copy_from_slice(table_key.as_ref()), row);
1475 }
1476 }
1477}
1478
1479struct BatchTableInnerIterLogInner<S: StateStore, SD: ValueRowSerde> {
1481 iter: S::ChangeLogIter,
1483
1484 mapping: Arc<ColumnMapping>,
1485
1486 row_deserializer: Arc<SD>,
1487}
1488
1489impl<S: StateStore, SD: ValueRowSerde> BatchTableInnerIterLogInner<S, SD> {
1490 #[allow(clippy::too_many_arguments)]
1492 async fn new(
1493 store: &S,
1494 mapping: Arc<ColumnMapping>,
1495 row_deserializer: Arc<SD>,
1496 table_key_range: TableKeyRange,
1497 read_options: ReadLogOptions,
1498 start_epoch: u64,
1499 end_epoch: HummockReadEpoch,
1500 ) -> StorageResult<Self> {
1501 store
1502 .try_wait_epoch(
1503 end_epoch,
1504 TryWaitEpochOptions {
1505 table_id: read_options.table_id,
1506 },
1507 )
1508 .await?;
1509 let iter = store
1510 .iter_log(
1511 (
1512 start_epoch,
1513 end_epoch
1514 .get_epoch()
1515 .expect("change-log reads require an explicit epoch"),
1516 ),
1517 table_key_range,
1518 read_options,
1519 )
1520 .await?;
1521 let iter = Self {
1522 iter,
1523 mapping,
1524 row_deserializer,
1525 };
1526 Ok(iter)
1527 }
1528
1529 fn into_stream<K: CopyFromSlice>(self) -> impl Stream<Item = StorageResult<(K, ChangeLogRow)>> {
1531 self.iter.into_stream(move |(table_key, value)| {
1532 value
1533 .try_map(|value| {
1534 let full_row = self.row_deserializer.deserialize(value)?;
1535 let row = self
1536 .mapping
1537 .project(OwnedRow::new(full_row))
1538 .into_owned_row();
1539 Ok(row)
1540 })
1541 .map(|row| (K::copy_from_slice(table_key.as_ref()), row))
1542 })
1543 }
1544}