risingwave_meta/manager/iceberg_compaction/
mod.rs

1// Copyright 2026 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod gc;
16mod manual;
17mod schedule;
18mod stream;
19
20use std::collections::{BTreeMap, HashMap, HashSet};
21use std::sync::Arc;
22use std::time::Duration;
23
24use anyhow::anyhow;
25use parking_lot::RwLock;
26use risingwave_common::id::WorkerId;
27use risingwave_connector::sink::SinkParam;
28use risingwave_connector::sink::catalog::{SinkCatalog, SinkId};
29use risingwave_connector::sink::iceberg::{ICEBERG_SINK, IcebergConfig};
30use risingwave_connector::source::UPSTREAM_SOURCE_KEY;
31use risingwave_pb::iceberg_compaction::SubscribeIcebergCompactionEventRequest;
32use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
33use tonic::Streaming;
34
35use super::MetaSrvEnv;
36use crate::MetaResult;
37use crate::controller::streaming_job::AbortCreatingJobResult;
38use crate::hummock::IcebergCompactorManagerRef;
39use crate::manager::MetadataManager;
40use crate::rpc::metrics::MetaMetrics;
41
42pub type IcebergCompactionManagerRef = Arc<IcebergCompactionManager>;
43
44pub(crate) type CompactorChangeTx =
45    UnboundedSender<(WorkerId, Streaming<SubscribeIcebergCompactionEventRequest>)>;
46
47pub(crate) type CompactorChangeRx =
48    UnboundedReceiver<(WorkerId, Streaming<SubscribeIcebergCompactionEventRequest>)>;
49
50type ManualCompactionWaiter = tokio::sync::oneshot::Sender<MetaResult<u64>>;
51
52use schedule::CompactionTrack;
53pub use schedule::IcebergCompactionScheduleStatus;
54
55pub struct IcebergCompactionManager {
56    pub env: MetaSrvEnv,
57    inner: Arc<RwLock<IcebergCompactionManagerInner>>,
58
59    metadata_manager: MetadataManager,
60    pub iceberg_compactor_manager: IcebergCompactorManagerRef,
61
62    compactor_streams_change_tx: CompactorChangeTx,
63
64    pub metrics: Arc<MetaMetrics>,
65}
66
67struct IcebergCompactionManagerInner {
68    sink_schedules: HashMap<SinkId, CompactionTrack>,
69    snapshot_expiration_sink_ids: HashSet<SinkId>,
70    manual_compaction_waiters: HashMap<SinkId, ManualCompactionWaiter>,
71}
72
73impl IcebergCompactionManager {
74    fn report_timeout(&self) -> Duration {
75        Duration::from_secs(self.env.opts.iceberg_compaction_report_timeout_sec)
76    }
77
78    fn config_refresh_interval(&self) -> Duration {
79        Duration::from_secs(self.env.opts.iceberg_compaction_config_refresh_interval_sec)
80    }
81
82    pub fn build(
83        env: MetaSrvEnv,
84        metadata_manager: MetadataManager,
85        iceberg_compactor_manager: IcebergCompactorManagerRef,
86        metrics: Arc<MetaMetrics>,
87    ) -> (Arc<Self>, CompactorChangeRx) {
88        let (compactor_streams_change_tx, compactor_streams_change_rx) =
89            tokio::sync::mpsc::unbounded_channel();
90        (
91            Arc::new(Self {
92                env,
93                inner: Arc::new(RwLock::new(IcebergCompactionManagerInner {
94                    sink_schedules: HashMap::default(),
95                    snapshot_expiration_sink_ids: HashSet::default(),
96                    manual_compaction_waiters: HashMap::default(),
97                })),
98                metadata_manager,
99                iceberg_compactor_manager,
100                compactor_streams_change_tx,
101                metrics,
102            }),
103            compactor_streams_change_rx,
104        )
105    }
106
107    async fn get_sink_param(&self, sink_id: SinkId) -> MetaResult<SinkParam> {
108        let prost_sink_catalog = self
109            .metadata_manager
110            .catalog_controller
111            .get_sink_by_id(sink_id)
112            .await?
113            .ok_or_else(|| anyhow!("Sink not found: {}", sink_id))?;
114        let sink_catalog = SinkCatalog::from(prost_sink_catalog);
115        let param = SinkParam::try_from_sink_catalog(sink_catalog)?;
116        Ok(param)
117    }
118
119    async fn load_iceberg_config(&self, sink_id: SinkId) -> MetaResult<IcebergConfig> {
120        let sink_param = self.get_sink_param(sink_id).await?;
121        let iceberg_config = IcebergConfig::from_btreemap(sink_param.properties)?;
122        Ok(iceberg_config)
123    }
124
125    /// Clear the iceberg maintenance state of the sink aborted by
126    /// `try_abort_creating_streaming_job`, if any.
127    pub fn clear_maintenance_for_aborted_job(&self, abort_result: &AbortCreatingJobResult) {
128        if let Some(sink_id) = abort_result.aborted_sink_id {
129            self.clear_iceberg_maintenance_by_sink_id(sink_id);
130        }
131    }
132}
133
134/// User-created iceberg sinks have arbitrary names, so identify them by the
135/// connector property instead of the `__iceberg_sink_` name prefix.
136pub fn is_iceberg_sink(properties: &BTreeMap<String, String>) -> bool {
137    properties
138        .get(UPSTREAM_SOURCE_KEY)
139        .is_some_and(|connector| connector.eq_ignore_ascii_case(ICEBERG_SINK))
140}
risingwave_meta/manager/iceberg_compaction/mod.rs

risingwave_meta/manager/iceberg_compaction/
mod.rs
