Trait SplitEnumerator

pub trait SplitEnumerator: Sized + Send {
    type Split: SplitMetaData + Send;
    type Properties;

    // Required methods
    fn new<'async_trait>(
        properties: Self::Properties,
        context: SourceEnumeratorContextRef,
    ) -> Pin<Box<dyn Future<Output = Result<Self>> + Send + 'async_trait>>
       where Self: 'async_trait;
    fn list_splits<'life0, 'async_trait>(
        &'life0 mut self,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<Self::Split>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;

    // Provided methods
    fn on_drop_fragments<'life0, 'async_trait>(
        &'life0 mut self,
        _fragment_ids: Vec<u32>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn on_finish_backfill<'life0, 'async_trait>(
        &'life0 mut self,
        _fragment_ids: Vec<u32>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
}

SplitEnumerator fetches the split metadata from the external source service. NOTE: It runs in the meta server, so probably it should be moved to the meta crate.

Required Associated Types

type Split: SplitMetaData + Send

type Properties

Required Methods

fn new<'async_trait>( properties: Self::Properties, context: SourceEnumeratorContextRef, ) -> Pin<Box<dyn Future<Output = Result<Self>> + Send + 'async_trait>>

where Self: 'async_trait,

fn list_splits<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<Vec<Self::Split>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Provided Methods

fn on_drop_fragments<'life0, 'async_trait>( &'life0 mut self, _fragment_ids: Vec<u32>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Do some cleanup work when a fragment is dropped, e.g., drop Kafka consumer group.

fn on_finish_backfill<'life0, 'async_trait>( &'life0 mut self, _fragment_ids: Vec<u32>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Do some cleanup work when a backfill fragment is finished, e.g., drop Kafka consumer group.

Object Safety

This trait is not object safe.

Implementors

impl SplitEnumerator for DatagenSplitEnumerator

type Properties = DatagenProperties

type Split = DatagenSplit

impl SplitEnumerator for LegacyS3SplitEnumerator

type Properties = LegacyS3Properties

type Split = LegacyFsSplit

impl SplitEnumerator for PubsubSplitEnumerator

type Properties = PubsubProperties

type Split = PubsubSplit

impl SplitEnumerator for IcebergSplitEnumerator

type Properties = IcebergProperties

type Split = IcebergSplit

impl SplitEnumerator for KafkaSplitEnumerator

type Properties = KafkaProperties

type Split = KafkaSplit

impl SplitEnumerator for KinesisSplitEnumerator

type Properties = KinesisProperties

type Split = KinesisSplit

impl SplitEnumerator for MqttSplitEnumerator

type Properties = MqttProperties

type Split = MqttSplit

impl SplitEnumerator for NatsSplitEnumerator

type Properties = NatsProperties

type Split = NatsSplit

impl SplitEnumerator for NexmarkSplitEnumerator

type Properties = NexmarkProperties

type Split = NexmarkSplit

impl SplitEnumerator for PulsarSplitEnumerator

type Properties = PulsarProperties

type Split = PulsarSplit

impl SplitEnumerator for TestSourceSplitEnumerator

type Properties = TestSourceProperties

type Split = TestSourceSplit

impl<Src: OpendalSource> SplitEnumerator for OpendalEnumerator<Src>

type Properties = <Src as OpendalSource>::Properties

type Split = OpendalFsSplit<Src>

impl<T: CdcSourceTypeTrait> SplitEnumerator for DebeziumSplitEnumerator<T>

where Self: ListCdcSplits<CdcSourceType = T>,

type Properties = CdcProperties<T>

type Split = DebeziumCdcSplit<T>