Struct StreamChunk

pub struct StreamChunk {
    ops: Arc<[Op]>,
    data: DataChunk,
}

StreamChunk is used to pass data over the streaming pathway.

Fields

ops: Arc<[Op]>

data: DataChunk

Implementations

impl StreamChunk

pub fn new(ops: impl Into<Arc<[Op]>>, columns: Vec<ArrayRef>) -> Self

Create a new StreamChunk with given ops and columns.

pub fn with_visibility( ops: impl Into<Arc<[Op]>>, columns: Vec<ArrayRef>, visibility: Bitmap, ) -> Self

Create a new StreamChunk with given ops, columns and visibility.

pub fn from_rows(rows: &[(Op, impl Row)], data_types: &[DataType]) -> Self

Build a StreamChunk from rows.

Panics if the rows is empty.

Should prefer using StreamChunkBuilder instead to avoid unnecessary allocation of rows.

pub fn data_chunk(&self) -> &DataChunk

Get the reference of the underlying data chunk.

pub fn compact(self) -> Self

compact the StreamChunk with its visibility map

pub fn split(&self, size: usize) -> Vec<Self>

Split the StreamChunk into multiple chunks with the given size at most.

When the total cardinality of all the chunks is not evenly divided by the size, the last new chunk will be the remainder.

For consecutive UpdateDelete and UpdateInsert, they will be kept in one chunk. As a result, some chunks may have size + 1 rows.

pub fn into_parts(self) -> (DataChunk, Arc<[Op]>)

pub fn from_parts(ops: impl Into<Arc<[Op]>>, data_chunk: DataChunk) -> Self

pub fn into_inner(self) -> (Arc<[Op]>, Vec<ArrayRef>, Bitmap)

pub fn to_protobuf(&self) -> PbStreamChunk

pub fn from_protobuf(prost: &PbStreamChunk) -> ArrayResult<Self>

pub fn ops(&self) -> &[Op]

pub fn to_pretty(&self) -> impl Display

Returns a table-like text representation of the StreamChunk.

pub fn to_pretty_with_schema(&self, schema: &Schema) -> impl Display

Returns a table-like text representation of the StreamChunk with a header of column names from the given schema.

fn to_pretty_inner(&self, schema: Option<&Schema>) -> impl Display

pub fn project(&self, indices: &[usize]) -> Self

Reorder (and possibly remove) columns.

e.g. if indices is [2, 1, 0], and the chunk contains column [a, b, c], then the output will be [c, b, a]. If indices is [2, 0], then the output will be [c, a]. If the input mapping is identity mapping, no reorder will be performed.

pub fn eliminate_adjacent_noop_update(self) -> Self

Remove the adjacent delete-insert if their row value are the same.

pub fn project_with_vis(&self, indices: &[usize], vis: Bitmap) -> Self

Reorder columns and set visibility.

pub fn clone_with_vis(&self, vis: Bitmap) -> Self

Clone the StreamChunk with a new visibility.

impl StreamChunk

pub fn records(&self) -> StreamChunkRefIter<'_>

Return an iterator on stream records of this stream chunk.

pub fn rows(&self) -> impl Iterator<Item = (Op, RowRef<'_>)>

Return an iterator on rows of this stream chunk.

Should consider using StreamChunk::records if possible.

pub fn rows_in( &self, range: Range<usize>, ) -> impl Iterator<Item = (Op, RowRef<'_>)>

Return an iterator on rows of this stream chunk in a range.

pub fn row_at(&self, pos: usize) -> (Op, RowRef<'_>, bool)

Random access a row at pos. Return the op, data and whether the row is visible.

pub fn rows_with_holes(&self) -> impl Iterator<Item = Option<(Op, RowRef<'_>)>>

Methods from Deref<Target = DataChunk>

pub(crate) const PRETTY_TABLE_PRESET: &'static str = "||--+-++| ++++++"

pub fn next_visible_row_idx(&self, row_idx: usize) -> Option<usize>

Return the next visible row index on or after row_idx.

pub fn dimension(&self) -> usize

pub fn cardinality(&self) -> usize

cardinality returns the number of visible tuples

pub fn capacity(&self) -> usize

capacity returns physical length of any chunk column

pub fn selectivity(&self) -> f64

pub fn with_visibility(&self, visibility: impl Into<Bitmap>) -> Self

pub fn visibility(&self) -> &Bitmap

pub fn set_visibility(&mut self, visibility: Bitmap)

pub fn is_compacted(&self) -> bool

pub fn column_at(&self, idx: usize) -> &ArrayRef

pub fn columns(&self) -> &[ArrayRef]

pub fn data_types(&self) -> Vec<DataType>

Returns the data types of all columns.

pub fn split_column_at(&self, idx: usize) -> (Self, Self)

Divides one chunk into two at an column index.

Panics

Panics if idx > columns.len().

pub fn to_protobuf(&self) -> PbDataChunk

pub fn compact_cow(&self) -> Cow<'_, Self>

Convert the chunk to compact format.

If the chunk is not compacted, return a new compacted chunk, otherwise return a reference to self.

pub fn get_hash_values<H: BuildHasher>( &self, column_idxes: &[usize], hasher_builder: H, ) -> Vec<HashCode<H>>

Compute hash values for each row. The number of the returning HashCodes is self.capacity(). When skip_invisible_row is true, the HashCode for the invisible rows is arbitrary.

pub fn row_at(&self, pos: usize) -> (RowRef<'_>, bool)

Random access a tuple in a data chunk. Return in a row format.

Arguments

• pos - Index of look up tuple
• RowRef - Reference of data tuple
• bool - whether this tuple is visible

pub fn row_at_unchecked_vis(&self, pos: usize) -> RowRef<'_>

Random access a tuple in a data chunk. Return in a row format. Note that this function do not return whether the row is visible.

Arguments

• pos - Index of look up tuple

pub fn to_pretty(&self) -> impl Display

Returns a table-like text representation of the DataChunk.

pub fn keep_columns(&self, column_indices: &[usize]) -> Self

Keep the specified columns and set the rest elements to null.

Example

i i i                            i i i
1 2 3  --> keep_columns([1]) --> . 2 .
4 5 6                            . 5 .

pub fn project(&self, indices: &[usize]) -> Self

Reorder (and possibly remove) columns.

e.g. if indices is [2, 1, 0], and the chunk contains column [a, b, c], then the output will be [c, b, a]. If indices is [2, 0], then the output will be [c, a]. If the input mapping is identity mapping, no reorder will be performed.

pub fn project_with_vis(&self, indices: &[usize], visibility: Bitmap) -> Self

Reorder columns and set visibility.

pub fn reorder_rows(&self, indexes: &[usize]) -> Self

Reorder rows by indexes.

fn partition_sizes(&self) -> (usize, Vec<&ArrayRef>)

Partition fixed size datums and variable length ones.

In some cases, we have fixed size for the entire column, when the datatypes are fixed size or the datums are constants. As such we can compute the size for it just once for the column.

Otherwise, for variable sized datatypes, such as varchar, we have to individually compute their sizes per row.

pub fn serialize(&self) -> Vec<Bytes>

Serialize each row into value encoding bytes.

The returned vector’s size is self.capacity() and for the invisible row will give a empty bytes.

pub fn serialize_with(&self, serializer: &impl ValueRowSerializer) -> Vec<Bytes>

Serialize each row into bytes with given serializer.

This is similar to serialize but it uses a custom serializer. Prefer serialize if possible since it might be more efficient due to columnar operations.

pub fn estimate_value_encoding_size(&self, column_indices: &[usize]) -> usize

Estimate size of hash keys. Their indices in a row are indicated by column_indices. Size here refers to the number of u8s required to store the serialized datum.

pub fn rows(&self) -> DataChunkRefIter<'_>

Get an iterator for visible rows.

pub fn rows_in(&self, range: Range<usize>) -> DataChunkRefIter<'_>

Get an iterator for visible rows in range.

pub fn rows_with_holes(&self) -> DataChunkRefIterWithHoles<'_>

Get an iterator for all rows in the chunk, and a None represents an invisible row.

Trait Implementations

impl Clone for StreamChunk

fn clone(&self) -> StreamChunk

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StreamChunk

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for StreamChunk

fn default() -> Self

Create a 0-row-0-col StreamChunk. Only used in some existing tests. This is NOT the same as an empty chunk, which has 0 rows but with columns aligned with executor schema.

impl Deref for StreamChunk

type Target = DataChunk

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.

impl DerefMut for StreamChunk

fn deref_mut(&mut self) -> &mut Self::Target

Mutably dereferences the value.

impl EstimateSize for StreamChunk

fn estimated_heap_size(&self) -> usize

The estimated heap size of the current struct in bytes.

fn estimated_size(&self) -> usize

where Self: Sized,

The estimated total size of the current struct in bytes, including the estimated_heap_size and the size of Self.

impl From<DataChunk> for StreamChunk

StreamChunk can be created from DataChunk with all operations set to Insert.

fn from(data: DataChunk) -> Self

Converts to this type from the input type.

impl From<StreamChunk> for StreamChunkMut

fn from(c: StreamChunk) -> Self

Converts to this type from the input type.

impl From<StreamChunkMut> for StreamChunk

fn from(c: StreamChunkMut) -> Self

Converts to this type from the input type.

impl PartialEq for StreamChunk

fn eq(&self, other: &StreamChunk) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl StreamChunkTestExt for StreamChunk

fn from_pretty(s: &str) -> Self

Parse a chunk from string.

fn valid(&self) -> bool

Validate the StreamChunk layout.

fn concat(chunks: Vec<StreamChunk>) -> StreamChunk

Concatenate multiple StreamChunk into one.

fn sort_rows(self) -> Self

Sort rows.

fn gen_stream_chunks( num_of_chunks: usize, chunk_size: usize, data_types: &[DataType], varchar_properties: &VarcharProperty, ) -> Vec<StreamChunk>

Generate num_of_chunks data chunks with type data_types, where each data chunk has cardinality of chunk_size. TODO(kwannoel): Generate different types of op, different vis.

fn gen_stream_chunks_inner( num_of_chunks: usize, chunk_size: usize, data_types: &[DataType], varchar_properties: &VarcharProperty, visibility_percent: f64, inserts_percent: f64, ) -> Vec<StreamChunk>

impl StructuralPartialEq for StreamChunk