Struct OpenDalSinkWriter 

pub struct OpenDalSinkWriter {
    schema: SchemaRef,
    operator: Operator,
    sink_writer: Option<FileWriterEnum>,
    write_path: String,
    executor_id: u64,
    encode_type: SinkEncode,
    row_encoder: JsonEncoder,
    engine_type: EngineType,
    pub(crate) batching_strategy: BatchingStrategy,
    current_bached_row_num: usize,
    created_time: SystemTime,
}

Fields

schema: SchemaRef

operator: Operator

sink_writer: Option<FileWriterEnum>

write_path: String

executor_id: u64

encode_type: SinkEncode

row_encoder: JsonEncoder

engine_type: EngineType

batching_strategy: BatchingStrategy

current_bached_row_num: usize

created_time: SystemTime

Implementations

impl OpenDalSinkWriter

Public interface exposed to BatchingLogSinker, used to write chunk and commit files.

pub async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()>

This method writes a chunk.

pub async fn commit(&mut self) -> Result<bool>

This method close current writer, finish writing a file and returns whether the commit is successful.

pub async fn try_commit(&mut self) -> Result<bool>

impl OpenDalSinkWriter

Private methods related to batching.

fn can_commit(&self) -> bool

Method for judging whether batch condition is met.

fn path_partition_prefix(&self, duration: &Duration) -> String

fn duration_seconds_since_writer_created(&self) -> usize

async fn append_only(&mut self, chunk: StreamChunk) -> Result<()>

impl OpenDalSinkWriter

Init methods.

pub fn new( operator: Operator, write_path: &str, rw_schema: Schema, executor_id: u64, format_desc: &SinkFormatDesc, engine_type: EngineType, batching_strategy: BatchingStrategy, ) -> Result<Self>

async fn create_object_writer(&mut self) -> Result<OpendalWriter>

async fn create_sink_writer(&mut self) -> Result<()>