Skip to main content

risingwave_connector/connector_common/iceberg/
mock_catalog.rs

1// Copyright 2024 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16
17use async_trait::async_trait;
18use iceberg::io::FileIO;
19use iceberg::spec::{
20    NestedField, PrimitiveType, Schema, TableMetadataBuilder, Transform, Type,
21    UnboundPartitionField, UnboundPartitionSpec,
22};
23use iceberg::table::Table;
24use iceberg::{
25    Catalog as CatalogV2, Namespace, NamespaceIdent, Runtime, TableCommit, TableCreation,
26    TableIdent,
27};
28
29#[cfg(any(test, feature = "test"))]
30pub use self::test_tables::MockTableGuard;
31
32#[cfg(any(test, feature = "test"))]
33mod test_tables {
34    use std::collections::HashMap;
35    use std::collections::hash_map::Entry;
36    use std::sync::LazyLock;
37
38    use iceberg::io::FileIO;
39    use iceberg::spec::{
40        FormatVersion, NestedField, PrimitiveType, Schema, SortOrder, TableMetadataBuilder, Type,
41        UnboundPartitionSpec,
42    };
43    use iceberg::table::Table;
44    use iceberg::{NamespaceIdent, Runtime, TableCommit, TableIdent};
45    use parking_lot::Mutex;
46
47    use super::MockCatalog;
48
49    static TABLES: LazyLock<Mutex<HashMap<TableIdent, Table>>> =
50        LazyLock::new(|| Mutex::new(HashMap::new()));
51
52    /// A registered table that is removed from the mock catalog when dropped.
53    #[must_use]
54    pub struct MockTableGuard {
55        ident: TableIdent,
56    }
57
58    impl MockTableGuard {
59        pub fn update(&self, table: Table) {
60            assert_eq!(table.identifier(), &self.ident);
61            *TABLES.lock().get_mut(&self.ident).unwrap() = table;
62        }
63    }
64
65    impl Drop for MockTableGuard {
66        fn drop(&mut self) {
67            TABLES.lock().remove(&self.ident);
68        }
69    }
70
71    impl MockCatalog {
72        /// Register a table shared by mock catalog instances. Use distinct identifiers in parallel tests.
73        pub fn register(table: Table) -> MockTableGuard {
74            let ident = table.identifier().clone();
75            match TABLES.lock().entry(ident.clone()) {
76                Entry::Vacant(entry) => {
77                    entry.insert(table);
78                }
79                Entry::Occupied(_) => panic!("mock table {ident} is already registered"),
80            }
81            MockTableGuard { ident }
82        }
83
84        /// Registered tables and the built-in `updates_v2`/`updates_v3` tables.
85        pub(super) fn test_table(ident: &TableIdent) -> Option<Table> {
86            if let Some(table) = TABLES.lock().get(ident).cloned() {
87                return Some(table);
88            }
89            match ident.name() {
90                "updates_v2" => Some(Self::updates_table(ident.name(), FormatVersion::V2)),
91                "updates_v3" => Some(Self::updates_table(ident.name(), FormatVersion::V3)),
92                _ => None,
93            }
94        }
95
96        pub(super) fn test_table_exists(ident: &TableIdent) -> bool {
97            TABLES.lock().contains_key(ident) || matches!(ident.name(), "updates_v2" | "updates_v3")
98        }
99
100        /// Apply a commit to a registered table. Returns the commit back if the table is not
101        /// registered.
102        pub(super) fn update_test_table(
103            commit: TableCommit,
104        ) -> Result<iceberg::Result<Table>, Box<TableCommit>> {
105            let mut tables = TABLES.lock();
106            let Some(table) = tables.get_mut(commit.identifier()) else {
107                return Err(Box::new(commit));
108            };
109            Ok(apply_commit(table, commit))
110        }
111
112        pub(super) fn updates_table(name: &str, version: FormatVersion) -> Table {
113            let schema = Schema::builder()
114                .with_fields(vec![
115                    NestedField::optional(10, "id", Type::Primitive(PrimitiveType::Int)).into(),
116                    NestedField::optional(20, "value", Type::Primitive(PrimitiveType::Long)).into(),
117                    NestedField::optional(30, "_row_id", Type::Primitive(PrimitiveType::Long))
118                        .into(),
119                ])
120                .build()
121                .unwrap();
122            let metadata = TableMetadataBuilder::new(
123                schema,
124                UnboundPartitionSpec::builder().build(),
125                SortOrder::unsorted_order(),
126                "memory://updates".to_owned(),
127                version,
128                HashMap::new(),
129            )
130            .unwrap()
131            .build()
132            .unwrap()
133            .metadata;
134            // No writer properties, like tables created by older PK-index sinks.
135            Table::builder()
136                .identifier(TableIdent::new(
137                    NamespaceIdent::new("mock_namespace".to_owned()),
138                    name.to_owned(),
139                ))
140                .file_io(FileIO::new_with_memory())
141                .runtime(Runtime::try_current().unwrap())
142                .metadata(metadata)
143                .build()
144                .unwrap()
145        }
146    }
147
148    fn apply_commit(table: &mut Table, mut commit: TableCommit) -> iceberg::Result<Table> {
149        for requirement in commit.take_requirements() {
150            requirement.check(Some(table.metadata()))?;
151        }
152        let mut metadata_builder = table
153            .metadata()
154            .clone()
155            .into_builder(table.metadata_location().map(str::to_owned));
156        for update in commit.take_updates() {
157            metadata_builder = update.apply(metadata_builder)?;
158        }
159        let mut builder = Table::builder()
160            .identifier(table.identifier().clone())
161            .file_io(table.file_io().clone())
162            .runtime(Runtime::try_current().unwrap())
163            .metadata(metadata_builder.build()?.metadata);
164        if let Some(location) = table.metadata_location() {
165            builder = builder.metadata_location(location);
166        }
167        let updated = builder.build()?;
168        *table = updated.clone();
169        Ok(updated)
170    }
171}
172
173/// A mock catalog for Iceberg planner and executor tests.
174#[derive(Debug)]
175pub struct MockCatalog;
176
177impl MockCatalog {
178    const RANGE_TABLE: &'static str = "range_table";
179    const SPARSE_TABLE: &'static str = "sparse_table";
180}
181
182impl MockCatalog {
183    fn build_table(name: &str, schema: Schema, partition_spec: UnboundPartitionSpec) -> Table {
184        let file_io = FileIO::new_with_memory();
185        let table_creation = TableCreation {
186            name: "ignore".to_owned(),
187            location: Some("1".to_owned()),
188            schema,
189            partition_spec: Some(partition_spec),
190            sort_order: None,
191            properties: HashMap::new(),
192            format_version: iceberg::spec::FormatVersion::V2,
193        };
194        Table::builder()
195            .identifier(TableIdent::new(
196                NamespaceIdent::new("mock_namespace".to_owned()),
197                name.to_owned(),
198            ))
199            .file_io(file_io)
200            .runtime(Runtime::try_current().unwrap())
201            .metadata(
202                TableMetadataBuilder::from_table_creation(table_creation)
203                    .unwrap()
204                    .build()
205                    .unwrap()
206                    .metadata,
207            )
208            .build()
209            .unwrap()
210    }
211
212    fn sparse_table() -> Table {
213        Self::build_table(
214            Self::SPARSE_TABLE,
215            Schema::builder()
216                .with_fields(vec![
217                    NestedField::new(1, "v1", Type::Primitive(PrimitiveType::Int), true).into(),
218                    NestedField::new(2, "v2", Type::Primitive(PrimitiveType::Long), true).into(),
219                    NestedField::new(3, "v3", Type::Primitive(PrimitiveType::String), true).into(),
220                    NestedField::new(4, "v4", Type::Primitive(PrimitiveType::Time), true).into(),
221                ])
222                .build()
223                .unwrap(),
224            UnboundPartitionSpec::builder()
225                .with_spec_id(1)
226                .add_partition_fields(vec![
227                    UnboundPartitionField {
228                        source_id: 1,
229                        field_id: Some(5),
230                        name: "f1".to_owned(),
231                        transform: Transform::Identity,
232                    },
233                    UnboundPartitionField {
234                        source_id: 2,
235                        field_id: Some(6),
236                        name: "f2".to_owned(),
237                        transform: Transform::Bucket(1),
238                    },
239                    UnboundPartitionField {
240                        source_id: 3,
241                        field_id: Some(7),
242                        name: "f3".to_owned(),
243                        transform: Transform::Truncate(1),
244                    },
245                    UnboundPartitionField {
246                        source_id: 4,
247                        field_id: Some(8),
248                        name: "f4".to_owned(),
249                        transform: Transform::Void,
250                    },
251                ])
252                .unwrap()
253                .build(),
254        )
255    }
256
257    fn range_table() -> Table {
258        Self::build_table(
259            Self::RANGE_TABLE,
260            Schema::builder()
261                .with_fields(vec![
262                    NestedField::new(1, "v1", Type::Primitive(PrimitiveType::Date), true).into(),
263                    NestedField::new(2, "v2", Type::Primitive(PrimitiveType::Timestamp), true)
264                        .into(),
265                    NestedField::new(3, "v3", Type::Primitive(PrimitiveType::Timestamptz), true)
266                        .into(),
267                    NestedField::new(4, "v4", Type::Primitive(PrimitiveType::Timestamptz), true)
268                        .into(),
269                ])
270                .build()
271                .unwrap(),
272            UnboundPartitionSpec::builder()
273                .with_spec_id(1)
274                .add_partition_fields(vec![
275                    UnboundPartitionField {
276                        source_id: 1,
277                        field_id: Some(5),
278                        name: "f1".to_owned(),
279                        transform: Transform::Year,
280                    },
281                    UnboundPartitionField {
282                        source_id: 2,
283                        field_id: Some(6),
284                        name: "f2".to_owned(),
285                        transform: Transform::Month,
286                    },
287                    UnboundPartitionField {
288                        source_id: 3,
289                        field_id: Some(7),
290                        name: "f3".to_owned(),
291                        transform: Transform::Day,
292                    },
293                    UnboundPartitionField {
294                        source_id: 4,
295                        field_id: Some(8),
296                        name: "f4".to_owned(),
297                        transform: Transform::Hour,
298                    },
299                ])
300                .unwrap()
301                .build(),
302        )
303    }
304}
305
306#[async_trait]
307impl CatalogV2 for MockCatalog {
308    /// List namespaces from table.
309    async fn list_namespaces(
310        &self,
311        _parent: Option<&NamespaceIdent>,
312    ) -> iceberg::Result<Vec<NamespaceIdent>> {
313        todo!()
314    }
315
316    /// Create a new namespace inside the catalog.
317    async fn create_namespace(
318        &self,
319        _namespace: &iceberg::NamespaceIdent,
320        _properties: HashMap<String, String>,
321    ) -> iceberg::Result<iceberg::Namespace> {
322        todo!()
323    }
324
325    /// Get a namespace information from the catalog.
326    async fn get_namespace(&self, _namespace: &NamespaceIdent) -> iceberg::Result<Namespace> {
327        todo!()
328    }
329
330    /// Check if namespace exists in catalog.
331    async fn namespace_exists(&self, _namespace: &NamespaceIdent) -> iceberg::Result<bool> {
332        todo!()
333    }
334
335    /// Drop a namespace from the catalog.
336    async fn drop_namespace(&self, _namespace: &NamespaceIdent) -> iceberg::Result<()> {
337        todo!()
338    }
339
340    /// List tables from namespace.
341    async fn list_tables(&self, _namespace: &NamespaceIdent) -> iceberg::Result<Vec<TableIdent>> {
342        todo!()
343    }
344
345    async fn update_namespace(
346        &self,
347        _namespace: &NamespaceIdent,
348        _properties: HashMap<String, String>,
349    ) -> iceberg::Result<()> {
350        todo!()
351    }
352
353    /// Create a new table inside the namespace.
354    async fn create_table(
355        &self,
356        _namespace: &NamespaceIdent,
357        _creation: TableCreation,
358    ) -> iceberg::Result<Table> {
359        todo!()
360    }
361
362    /// Load table from the catalog.
363    async fn load_table(&self, table: &TableIdent) -> iceberg::Result<Table> {
364        #[cfg(any(test, feature = "test"))]
365        if let Some(table) = Self::test_table(table) {
366            return Ok(table);
367        }
368        match table.name.as_ref() {
369            Self::SPARSE_TABLE => Ok(Self::sparse_table()),
370            Self::RANGE_TABLE => Ok(Self::range_table()),
371            _ => unimplemented!("table {} not found", table.name()),
372        }
373    }
374
375    /// Drop a table from the catalog.
376    async fn drop_table(&self, _table: &TableIdent) -> iceberg::Result<()> {
377        todo!()
378    }
379
380    async fn purge_table(&self, table: &TableIdent) -> iceberg::Result<()> {
381        self.drop_table(table).await
382    }
383
384    /// Check if a table exists in the catalog.
385    async fn table_exists(&self, table: &TableIdent) -> iceberg::Result<bool> {
386        #[cfg(any(test, feature = "test"))]
387        if Self::test_table_exists(table) {
388            return Ok(true);
389        }
390        match table.name.as_ref() {
391            Self::SPARSE_TABLE => Ok(true),
392            Self::RANGE_TABLE => Ok(true),
393            _ => Ok(false),
394        }
395    }
396
397    /// Rename a table in the catalog.
398    async fn rename_table(&self, _src: &TableIdent, _dest: &TableIdent) -> iceberg::Result<()> {
399        todo!()
400    }
401
402    /// Update a table to the catalog.
403    async fn update_table(&self, commit: TableCommit) -> iceberg::Result<Table> {
404        #[cfg(any(test, feature = "test"))]
405        let commit = match Self::update_test_table(commit) {
406            Ok(result) => return result,
407            Err(commit) => *commit,
408        };
409        let _ = commit;
410        todo!()
411    }
412
413    #[expect(
414        clippy::disallowed_types,
415        reason = "iceberg catalog trait requires returning iceberg::Error"
416    )]
417    async fn register_table(
418        &self,
419        _table_ident: &TableIdent,
420        _metadata_location: String,
421    ) -> iceberg::Result<Table> {
422        Err(iceberg::Error::new(
423            iceberg::ErrorKind::Unexpected,
424            "register_table is not supported in mock catalog",
425        ))
426    }
427}
428
429// Iceberg tables require a native Tokio runtime, which madsim does not provide.
430#[cfg(all(test, not(madsim)))]
431mod tests {
432    use super::*;
433
434    #[tokio::test]
435    async fn registered_tables_are_isolated_and_scoped() -> iceberg::Result<()> {
436        use iceberg::spec::FormatVersion;
437
438        let first = MockCatalog::updates_table(
439            &format!("registered_{}", uuid::Uuid::new_v4().simple()),
440            FormatVersion::V2,
441        );
442        let second = MockCatalog::updates_table(
443            &format!("registered_{}", uuid::Uuid::new_v4().simple()),
444            FormatVersion::V2,
445        );
446        let first_guard = MockCatalog::register(first.clone());
447        let second_guard = MockCatalog::register(second.clone());
448        let catalog = MockCatalog;
449        assert!(catalog.table_exists(first.identifier()).await?);
450        assert!(catalog.table_exists(second.identifier()).await?);
451        assert_eq!(
452            catalog
453                .load_table(first.identifier())
454                .await?
455                .metadata()
456                .uuid(),
457            first.metadata().uuid()
458        );
459
460        let updated = MockCatalog::updates_table(first.identifier().name(), FormatVersion::V3);
461        first_guard.update(updated.clone());
462        assert_eq!(
463            catalog
464                .load_table(first.identifier())
465                .await?
466                .metadata()
467                .uuid(),
468            updated.metadata().uuid()
469        );
470        assert_eq!(
471            catalog
472                .load_table(second.identifier())
473                .await?
474                .metadata()
475                .uuid(),
476            second.metadata().uuid()
477        );
478        drop(first_guard);
479        assert!(!catalog.table_exists(first.identifier()).await?);
480        assert!(catalog.table_exists(second.identifier()).await?);
481        drop(second_guard);
482        assert!(!catalog.table_exists(second.identifier()).await?);
483        Ok(())
484    }
485}