Skip to main content

risingwave_meta/controller/
utils.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeSet, HashMap, HashSet};
16use std::sync::Arc;
17
18use anyhow::{Context, anyhow};
19use itertools::Itertools;
20use risingwave_common::bitmap::Bitmap;
21use risingwave_common::catalog::{
22    FragmentTypeFlag, FragmentTypeMask, ICEBERG_SINK_PREFIX, ICEBERG_SOURCE_PREFIX,
23};
24use risingwave_common::hash::{ActorMapping, VnodeBitmapExt, WorkerSlotId, WorkerSlotMapping};
25use risingwave_common::id::{JobId, SubscriptionId};
26use risingwave_common::types::{DataType, Datum};
27use risingwave_common::util::value_encoding::DatumToProtoExt;
28use risingwave_common::util::worker_util::DEFAULT_RESOURCE_GROUP;
29use risingwave_common::{bail, hash};
30use risingwave_connector::WithOptionsSecResolved;
31use risingwave_connector::source::cdc::{CDC_COLUMN_EXCLUDE_LIST, parse_cdc_column_exclude_list};
32use risingwave_meta_model::fragment::DistributionType;
33use risingwave_meta_model::object::ObjectType;
34use risingwave_meta_model::prelude::*;
35use risingwave_meta_model::streaming_job::BackfillOrders;
36use risingwave_meta_model::table::TableType;
37use risingwave_meta_model::user_privilege::Action;
38use risingwave_meta_model::{
39    ActorId, ColumnCatalogArray, CreateType, DataTypeArray, DatabaseId, DispatcherType, FragmentId,
40    JobStatus, ObjectId, PrivilegeId, SchemaId, SinkId, SourceId, StreamNode, StreamSourceInfo,
41    TableId, TableIdArray, UserId, WorkerId, connection, database, fragment, fragment_relation,
42    function, index, object, object_dependency, schema, secret, sink, source, streaming_job,
43    subscription, table, user, user_default_privilege, user_privilege, view,
44};
45use risingwave_meta_model_migration::WithQuery;
46use risingwave_pb::catalog::{
47    PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource,
48    PbSubscription, PbTable, PbView,
49};
50use risingwave_pb::common::{PbObjectType, WorkerNode};
51use risingwave_pb::expr::{PbExprNode, expr_node};
52use risingwave_pb::meta::object::PbObjectInfo;
53use risingwave_pb::meta::subscribe_response::Info as NotificationInfo;
54use risingwave_pb::meta::{
55    ObjectDependency as PbObjectDependency, PbFragmentWorkerSlotMapping, PbObject, PbObjectGroup,
56};
57use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
58use risingwave_pb::plan_common::{ColumnCatalog, DefaultColumnDesc};
59use risingwave_pb::secret::PbSecretRef;
60use risingwave_pb::secret::secret_ref::PbRefAsType;
61use risingwave_pb::stream_plan::{PbDispatchOutputMapping, PbDispatcher, PbDispatcherType};
62use risingwave_pb::user::grant_privilege::{PbActionWithGrantOption, PbObject as PbGrantObject};
63use risingwave_pb::user::{PbAction, PbGrantPrivilege, PbUserInfo};
64use risingwave_sqlparser::ast::{
65    CstyleEscapedString, Ident, ObjectName, SecretRefAsType, SecretRefValue, SqlOption,
66    SqlOptionValue, Statement as SqlStatement, Value,
67};
68use risingwave_sqlparser::parser::Parser;
69use sea_orm::sea_query::{
70    Alias, CommonTableExpression, Expr, OnConflict, Query, QueryStatementBuilder, SelectStatement,
71    UnionType, WithClause,
72};
73use sea_orm::{
74    ColumnTrait, ConnectionTrait, DatabaseTransaction, DerivePartialModel, EntityTrait,
75    FromQueryResult, IntoActiveModel, JoinType, Order, PaginatorTrait, QueryFilter, QuerySelect,
76    RelationTrait, Set, Statement,
77};
78use thiserror_ext::AsReport;
79use tracing::warn;
80
81use crate::barrier::SharedFragmentInfo;
82use crate::controller::ObjectModel;
83use crate::controller::fragment::FragmentTypeMaskExt;
84use crate::controller::scale::resolve_streaming_job_definition;
85use crate::model::{FragmentDownstreamRelation, StreamContext};
86use crate::{MetaError, MetaResult};
87
88/// This function will construct a query using recursive cte to find all objects[(id, `obj_type`)] that are used by the given object.
89///
90/// # Examples
91///
92/// ```
93/// use risingwave_meta::controller::utils::construct_obj_dependency_query;
94/// use sea_orm::sea_query::*;
95/// use sea_orm::*;
96///
97/// let query = construct_obj_dependency_query(1.into());
98///
99/// assert_eq!(
100///     query.to_string(MysqlQueryBuilder),
101///     r#"WITH RECURSIVE `used_by_object_ids` (`used_by`) AS (SELECT `used_by` FROM `object_dependency` WHERE `object_dependency`.`oid` = 1 UNION ALL (SELECT `oid` FROM `object` WHERE `object`.`database_id` = 1 OR `object`.`schema_id` = 1) UNION ALL (SELECT `object_dependency`.`used_by` FROM `object_dependency` INNER JOIN `used_by_object_ids` ON `used_by_object_ids`.`used_by` = `oid`)) SELECT DISTINCT `oid`, `obj_type`, `schema_id`, `database_id` FROM `used_by_object_ids` INNER JOIN `object` ON `used_by_object_ids`.`used_by` = `oid` ORDER BY `oid` DESC"#
102/// );
103/// assert_eq!(
104///     query.to_string(PostgresQueryBuilder),
105///     r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL (SELECT "oid" FROM "object" WHERE "object"."database_id" = 1 OR "object"."schema_id" = 1) UNION ALL (SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid")) SELECT DISTINCT "oid", "obj_type", "schema_id", "database_id" FROM "used_by_object_ids" INNER JOIN "object" ON "used_by_object_ids"."used_by" = "oid" ORDER BY "oid" DESC"#
106/// );
107/// assert_eq!(
108///     query.to_string(SqliteQueryBuilder),
109///     r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL SELECT "oid" FROM "object" WHERE "object"."database_id" = 1 OR "object"."schema_id" = 1 UNION ALL SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid") SELECT DISTINCT "oid", "obj_type", "schema_id", "database_id" FROM "used_by_object_ids" INNER JOIN "object" ON "used_by_object_ids"."used_by" = "oid" ORDER BY "oid" DESC"#
110/// );
111/// ```
112pub fn construct_obj_dependency_query(obj_id: ObjectId) -> WithQuery {
113    let cte_alias = Alias::new("used_by_object_ids");
114    let cte_return_alias = Alias::new("used_by");
115
116    let mut base_query = SelectStatement::new()
117        .column(object_dependency::Column::UsedBy)
118        .from(ObjectDependency)
119        .and_where(object_dependency::Column::Oid.eq(obj_id))
120        .to_owned();
121
122    let belonged_obj_query = SelectStatement::new()
123        .column(object::Column::Oid)
124        .from(Object)
125        .and_where(
126            object::Column::DatabaseId
127                .eq(obj_id)
128                .or(object::Column::SchemaId.eq(obj_id)),
129        )
130        .to_owned();
131
132    let cte_referencing = Query::select()
133        .column((ObjectDependency, object_dependency::Column::UsedBy))
134        .from(ObjectDependency)
135        .inner_join(
136            cte_alias.clone(),
137            Expr::col((cte_alias.clone(), cte_return_alias.clone()))
138                .equals(object_dependency::Column::Oid),
139        )
140        .to_owned();
141
142    let mut common_table_expr = CommonTableExpression::new();
143    common_table_expr
144        .query(
145            base_query
146                .union(UnionType::All, belonged_obj_query)
147                .union(UnionType::All, cte_referencing)
148                .to_owned(),
149        )
150        .column(cte_return_alias.clone())
151        .table_name(cte_alias.clone());
152
153    SelectStatement::new()
154        .distinct()
155        .columns([
156            object::Column::Oid,
157            object::Column::ObjType,
158            object::Column::SchemaId,
159            object::Column::DatabaseId,
160        ])
161        .from(cte_alias.clone())
162        .inner_join(
163            Object,
164            Expr::col((cte_alias, cte_return_alias)).equals(object::Column::Oid),
165        )
166        .order_by(object::Column::Oid, Order::Desc)
167        .to_owned()
168        .with(
169            WithClause::new()
170                .recursive(true)
171                .cte(common_table_expr)
172                .to_owned(),
173        )
174}
175
176fn to_pb_object_type(obj_type: ObjectType) -> PbObjectType {
177    match obj_type {
178        ObjectType::Database => PbObjectType::Database,
179        ObjectType::Schema => PbObjectType::Schema,
180        ObjectType::Table => PbObjectType::Table,
181        ObjectType::Source => PbObjectType::Source,
182        ObjectType::Sink => PbObjectType::Sink,
183        ObjectType::View => PbObjectType::View,
184        ObjectType::Index => PbObjectType::Index,
185        ObjectType::Function => PbObjectType::Function,
186        ObjectType::Connection => PbObjectType::Connection,
187        ObjectType::Subscription => PbObjectType::Subscription,
188        ObjectType::Secret => PbObjectType::Secret,
189    }
190}
191
192/// Collect object dependencies with optional object id filtering, and optionally exclude non-created streaming jobs.
193async fn list_object_dependencies_impl(
194    txn: &DatabaseTransaction,
195    object_id: Option<ObjectId>,
196    include_creating: bool,
197) -> MetaResult<Vec<PbObjectDependency>> {
198    let referenced_alias = Alias::new("referenced_obj");
199    let mut query = ObjectDependency::find()
200        .select_only()
201        .columns([
202            object_dependency::Column::Oid,
203            object_dependency::Column::UsedBy,
204        ])
205        .column_as(
206            Expr::col((referenced_alias.clone(), object::Column::ObjType)),
207            "referenced_obj_type",
208        )
209        .join(
210            JoinType::InnerJoin,
211            object_dependency::Relation::Object1.def(),
212        )
213        .join_as(
214            JoinType::InnerJoin,
215            object_dependency::Relation::Object2.def(),
216            referenced_alias.clone(),
217        );
218    if let Some(object_id) = object_id {
219        query = query.filter(object_dependency::Column::UsedBy.eq(object_id));
220    }
221    let mut obj_dependencies: Vec<PbObjectDependency> = query
222        .into_tuple()
223        .all(txn)
224        .await?
225        .into_iter()
226        .map(|(oid, used_by, referenced_type)| PbObjectDependency {
227            object_id: used_by,
228            referenced_object_id: oid,
229            referenced_object_type: to_pb_object_type(referenced_type) as i32,
230        })
231        .collect();
232
233    let mut sink_query = Sink::find()
234        .select_only()
235        .columns([sink::Column::SinkId, sink::Column::TargetTable])
236        .filter(sink::Column::TargetTable.is_not_null());
237    if let Some(object_id) = object_id {
238        sink_query = sink_query.filter(
239            sink::Column::SinkId
240                .eq(object_id)
241                .or(sink::Column::TargetTable.eq(object_id)),
242        );
243    }
244    let sink_dependencies: Vec<(SinkId, TableId)> = sink_query.into_tuple().all(txn).await?;
245    obj_dependencies.extend(sink_dependencies.into_iter().map(|(sink_id, table_id)| {
246        PbObjectDependency {
247            object_id: table_id.into(),
248            referenced_object_id: sink_id.into(),
249            referenced_object_type: PbObjectType::Sink as i32,
250        }
251    }));
252
253    if !include_creating {
254        let mut streaming_job_ids = obj_dependencies
255            .iter()
256            .map(|dependency| dependency.object_id)
257            .collect_vec();
258        streaming_job_ids.sort_unstable();
259        streaming_job_ids.dedup();
260
261        if !streaming_job_ids.is_empty() {
262            let non_created_jobs: HashSet<JobId> = StreamingJob::find()
263                .select_only()
264                .columns([streaming_job::Column::JobId])
265                .filter(
266                    streaming_job::Column::JobId
267                        .is_in(streaming_job_ids)
268                        .and(streaming_job::Column::JobStatus.ne(JobStatus::Created)),
269                )
270                .into_tuple()
271                .all(txn)
272                .await?
273                .into_iter()
274                .collect();
275
276            if !non_created_jobs.is_empty() {
277                obj_dependencies.retain(|dependency| {
278                    !non_created_jobs.contains(&dependency.object_id.as_job_id())
279                });
280            }
281        }
282    }
283
284    Ok(obj_dependencies)
285}
286
287/// List object dependencies with optional filtering of non-created streaming jobs.
288pub async fn list_object_dependencies(
289    txn: &DatabaseTransaction,
290    include_creating: bool,
291) -> MetaResult<Vec<PbObjectDependency>> {
292    list_object_dependencies_impl(txn, None, include_creating).await
293}
294
295/// List object dependencies for the specified object id without filtering by job status.
296pub async fn list_object_dependencies_by_object_id(
297    txn: &DatabaseTransaction,
298    object_id: ObjectId,
299) -> MetaResult<Vec<PbObjectDependency>> {
300    list_object_dependencies_impl(txn, Some(object_id), true).await
301}
302
303/// This function will construct a query using recursive cte to find if dependent objects are already relying on the target table.
304///
305/// # Examples
306///
307/// ```
308/// use risingwave_meta::controller::utils::construct_sink_cycle_check_query;
309/// use sea_orm::sea_query::*;
310/// use sea_orm::*;
311///
312/// let query = construct_sink_cycle_check_query(1.into(), vec![2.into(), 3.into()]);
313///
314/// assert_eq!(
315///     query.to_string(MysqlQueryBuilder),
316///     r#"WITH RECURSIVE `used_by_object_ids_with_sink` (`oid`, `used_by`) AS (SELECT `oid`, `used_by` FROM `object_dependency` WHERE `object_dependency`.`oid` = 1 UNION ALL (SELECT `obj_dependency_with_sink`.`oid`, `obj_dependency_with_sink`.`used_by` FROM (SELECT `oid`, `used_by` FROM `object_dependency` UNION ALL (SELECT `sink_id`, `target_table` FROM `sink` WHERE `sink`.`target_table` IS NOT NULL)) AS `obj_dependency_with_sink` INNER JOIN `used_by_object_ids_with_sink` ON `used_by_object_ids_with_sink`.`used_by` = `obj_dependency_with_sink`.`oid` WHERE `used_by_object_ids_with_sink`.`used_by` <> `used_by_object_ids_with_sink`.`oid`)) SELECT COUNT(`used_by_object_ids_with_sink`.`used_by`) FROM `used_by_object_ids_with_sink` WHERE `used_by_object_ids_with_sink`.`used_by` IN (2, 3)"#
317/// );
318/// assert_eq!(
319///     query.to_string(PostgresQueryBuilder),
320///     r#"WITH RECURSIVE "used_by_object_ids_with_sink" ("oid", "used_by") AS (SELECT "oid", "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL (SELECT "obj_dependency_with_sink"."oid", "obj_dependency_with_sink"."used_by" FROM (SELECT "oid", "used_by" FROM "object_dependency" UNION ALL (SELECT "sink_id", "target_table" FROM "sink" WHERE "sink"."target_table" IS NOT NULL)) AS "obj_dependency_with_sink" INNER JOIN "used_by_object_ids_with_sink" ON "used_by_object_ids_with_sink"."used_by" = "obj_dependency_with_sink"."oid" WHERE "used_by_object_ids_with_sink"."used_by" <> "used_by_object_ids_with_sink"."oid")) SELECT COUNT("used_by_object_ids_with_sink"."used_by") FROM "used_by_object_ids_with_sink" WHERE "used_by_object_ids_with_sink"."used_by" IN (2, 3)"#
321/// );
322/// assert_eq!(
323///     query.to_string(SqliteQueryBuilder),
324///     r#"WITH RECURSIVE "used_by_object_ids_with_sink" ("oid", "used_by") AS (SELECT "oid", "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL SELECT "obj_dependency_with_sink"."oid", "obj_dependency_with_sink"."used_by" FROM (SELECT "oid", "used_by" FROM "object_dependency" UNION ALL SELECT "sink_id", "target_table" FROM "sink" WHERE "sink"."target_table" IS NOT NULL) AS "obj_dependency_with_sink" INNER JOIN "used_by_object_ids_with_sink" ON "used_by_object_ids_with_sink"."used_by" = "obj_dependency_with_sink"."oid" WHERE "used_by_object_ids_with_sink"."used_by" <> "used_by_object_ids_with_sink"."oid") SELECT COUNT("used_by_object_ids_with_sink"."used_by") FROM "used_by_object_ids_with_sink" WHERE "used_by_object_ids_with_sink"."used_by" IN (2, 3)"#
325/// );
326/// ```
327pub fn construct_sink_cycle_check_query(
328    target_table: ObjectId,
329    dependent_objects: Vec<ObjectId>,
330) -> WithQuery {
331    let cte_alias = Alias::new("used_by_object_ids_with_sink");
332    let depend_alias = Alias::new("obj_dependency_with_sink");
333
334    let mut base_query = SelectStatement::new()
335        .columns([
336            object_dependency::Column::Oid,
337            object_dependency::Column::UsedBy,
338        ])
339        .from(ObjectDependency)
340        .and_where(object_dependency::Column::Oid.eq(target_table))
341        .to_owned();
342
343    let query_sink_deps = SelectStatement::new()
344        .columns([sink::Column::SinkId, sink::Column::TargetTable])
345        .from(Sink)
346        .and_where(sink::Column::TargetTable.is_not_null())
347        .to_owned();
348
349    let cte_referencing = Query::select()
350        .column((depend_alias.clone(), object_dependency::Column::Oid))
351        .column((depend_alias.clone(), object_dependency::Column::UsedBy))
352        .from_subquery(
353            SelectStatement::new()
354                .columns([
355                    object_dependency::Column::Oid,
356                    object_dependency::Column::UsedBy,
357                ])
358                .from(ObjectDependency)
359                .union(UnionType::All, query_sink_deps)
360                .to_owned(),
361            depend_alias.clone(),
362        )
363        .inner_join(
364            cte_alias.clone(),
365            Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy))
366                .eq(Expr::col((depend_alias, object_dependency::Column::Oid))),
367        )
368        .and_where(
369            Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy)).ne(Expr::col((
370                cte_alias.clone(),
371                object_dependency::Column::Oid,
372            ))),
373        )
374        .to_owned();
375
376    let mut common_table_expr = CommonTableExpression::new();
377    common_table_expr
378        .query(base_query.union(UnionType::All, cte_referencing).to_owned())
379        .columns([
380            object_dependency::Column::Oid,
381            object_dependency::Column::UsedBy,
382        ])
383        .table_name(cte_alias.clone());
384
385    SelectStatement::new()
386        .expr(Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy)).count())
387        .from(cte_alias.clone())
388        .and_where(
389            Expr::col((cte_alias, object_dependency::Column::UsedBy)).is_in(dependent_objects),
390        )
391        .to_owned()
392        .with(
393            WithClause::new()
394                .recursive(true)
395                .cte(common_table_expr)
396                .to_owned(),
397        )
398}
399
400#[derive(Clone, DerivePartialModel, FromQueryResult, Debug)]
401#[sea_orm(entity = "Object")]
402pub struct PartialObject {
403    pub oid: ObjectId,
404    pub obj_type: ObjectType,
405    pub schema_id: Option<SchemaId>,
406    pub database_id: Option<DatabaseId>,
407}
408
409impl From<object::Model> for PartialObject {
410    fn from(object: object::Model) -> Self {
411        Self {
412            oid: object.oid,
413            obj_type: object.obj_type,
414            schema_id: object.schema_id,
415            database_id: object.database_id,
416        }
417    }
418}
419
420pub async fn get_belong_objects<C>(db: &C, object_id: ObjectId) -> MetaResult<Vec<object::Model>>
421where
422    C: ConnectionTrait,
423{
424    get_belong_objects_by_ids(db, [object_id]).await
425}
426
427/// Returns all objects that transitively belong to `object_ids`, excluding `object_ids`.
428pub async fn get_belong_objects_by_ids<C>(
429    db: &C,
430    object_ids: impl IntoIterator<Item = ObjectId>,
431) -> MetaResult<Vec<object::Model>>
432where
433    C: ConnectionTrait,
434{
435    let mut parents = object_ids.into_iter().collect_vec();
436    let mut visited = parents.iter().copied().collect::<HashSet<_>>();
437    let mut objects = vec![];
438    loop {
439        let children = Object::find()
440            .filter(object::Column::BelongToOid.is_in(parents))
441            .all(db)
442            .await?
443            .into_iter()
444            .filter(|object| visited.insert(object.oid))
445            .collect_vec();
446        if children.is_empty() {
447            break;
448        }
449
450        parents = children.iter().map(|object| object.oid).collect();
451        objects.extend(children);
452    }
453
454    Ok(objects)
455}
456
457#[derive(Clone, DerivePartialModel, FromQueryResult)]
458#[sea_orm(entity = "Fragment")]
459pub struct PartialFragmentStateTables {
460    pub fragment_id: FragmentId,
461    pub job_id: ObjectId,
462    pub state_table_ids: TableIdArray,
463}
464
465#[derive(Clone, Eq, PartialEq, Debug)]
466pub struct PartialActorLocation {
467    pub actor_id: ActorId,
468    pub fragment_id: FragmentId,
469    pub worker_id: WorkerId,
470}
471
472#[derive(FromQueryResult, Debug, Eq, PartialEq, Clone)]
473pub struct FragmentDesc {
474    pub fragment_id: FragmentId,
475    pub job_id: JobId,
476    pub fragment_type_mask: i32,
477    pub distribution_type: DistributionType,
478    pub state_table_ids: TableIdArray,
479    pub parallelism: i64,
480    pub vnode_count: i32,
481    pub stream_node: StreamNode,
482    pub parallelism_policy: String,
483}
484
485/// List all objects that are using the given one in a cascade way. It runs a recursive CTE to find all the dependencies.
486pub async fn get_referring_objects_cascade<C>(
487    obj_id: ObjectId,
488    object_type: ObjectType,
489    db: &C,
490) -> MetaResult<Vec<PartialObject>>
491where
492    C: ConnectionTrait,
493{
494    let query = construct_obj_dependency_query(obj_id);
495    let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
496    let mut objects = PartialObject::find_by_statement(Statement::from_sql_and_values(
497        db.get_database_backend(),
498        sql,
499        values,
500    ))
501    .all(db)
502    .await?;
503
504    let target_objects = std::iter::once((obj_id, object_type))
505        .chain(objects.iter().map(|object| (object.oid, object.obj_type)))
506        .collect_vec();
507    let mut existing_object_ids: HashSet<ObjectId> =
508        objects.iter().map(|object| object.oid).collect();
509    for (target_id, target_type) in target_objects {
510        let incoming_sink_objects =
511            get_incoming_sink_objects_for_target(target_id, target_type, db).await?;
512        for incoming_sink_object in incoming_sink_objects {
513            if existing_object_ids.insert(incoming_sink_object.oid) {
514                objects.push(incoming_sink_object);
515            }
516        }
517    }
518    Ok(objects)
519}
520
521/// Check if create a sink with given dependent objects into the target table will cause a cycle, return true if it will.
522pub async fn check_sink_into_table_cycle<C>(
523    target_table: ObjectId,
524    dependent_objs: Vec<ObjectId>,
525    db: &C,
526) -> MetaResult<bool>
527where
528    C: ConnectionTrait,
529{
530    if dependent_objs.is_empty() {
531        return Ok(false);
532    }
533
534    // special check for self referencing
535    if dependent_objs.contains(&target_table) {
536        return Ok(true);
537    }
538
539    let query = construct_sink_cycle_check_query(target_table, dependent_objs);
540    let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
541
542    let res = db
543        .query_one(Statement::from_sql_and_values(
544            db.get_database_backend(),
545            sql,
546            values,
547        ))
548        .await?
549        .unwrap();
550
551    let cnt: i64 = res.try_get_by(0)?;
552
553    Ok(cnt != 0)
554}
555
556/// Formats SQL options with secret values properly resolved.
557///
558/// This function processes configuration options that may contain sensitive data:
559/// - Plaintext options are directly converted to `SqlOption`.
560/// - Secret options are retrieved from the database and formatted as `SECRET {name}` without
561///   exposing the actual secret value.
562pub async fn format_with_option_secret_resolved(
563    txn: &DatabaseTransaction,
564    options_with_secret: &WithOptionsSecResolved,
565) -> MetaResult<Vec<SqlOption>> {
566    let mut options = Vec::new();
567    for (k, v) in options_with_secret.as_plaintext() {
568        let sql_option = SqlOption::try_from((k, v))
569            .map_err(|e| MetaError::invalid_parameter(e.to_report_string()))?;
570        options.push(sql_option);
571    }
572    for (name, secret_ref) in options_with_secret.as_secret() {
573        let secret_ref_value = resolve_secret_ref_value(txn, secret_ref).await?;
574        options.push(SqlOption::from_secret_ref(name, secret_ref_value));
575    }
576    Ok(options)
577}
578
579async fn resolve_secret_ref_value(
580    txn: &DatabaseTransaction,
581    secret_ref: &PbSecretRef,
582) -> MetaResult<SecretRefValue> {
583    let (secret, object) = Secret::find_by_id(secret_ref.secret_id)
584        .find_also_related(Object)
585        .one(txn)
586        .await?
587        .ok_or_else(|| MetaError::catalog_id_not_found("secret", secret_ref.secret_id))?;
588    let object =
589        object.ok_or_else(|| MetaError::catalog_id_not_found("object", secret_ref.secret_id))?;
590    let schema_id = object.schema_id.ok_or_else(|| {
591        MetaError::invalid_parameter(format!(
592            "secret {} does not belong to a schema",
593            secret_ref.secret_id
594        ))
595    })?;
596    let schema = Schema::find_by_id(schema_id)
597        .one(txn)
598        .await?
599        .ok_or_else(|| MetaError::catalog_id_not_found("schema", schema_id))?;
600
601    let ref_as = match PbRefAsType::try_from(secret_ref.ref_as) {
602        Ok(PbRefAsType::File) => SecretRefAsType::File,
603        Ok(PbRefAsType::Text | PbRefAsType::Unspecified) => SecretRefAsType::Text,
604        Err(_) => {
605            return Err(MetaError::invalid_parameter(format!(
606                "invalid reference type {} for secret {}",
607                secret_ref.ref_as, secret_ref.secret_id
608            )));
609        }
610    };
611
612    Ok(SecretRefValue {
613        secret_name: ObjectName(vec![
614            Ident::from_real_value(&schema.name),
615            Ident::from_real_value(&secret.name),
616        ]),
617        ref_as,
618    })
619}
620
621/// `ensure_object_id` ensures the existence of target object in the cluster.
622pub async fn ensure_object_id<C>(
623    object_type: ObjectType,
624    obj_id: impl Into<ObjectId>,
625    db: &C,
626) -> MetaResult<()>
627where
628    C: ConnectionTrait,
629{
630    let obj_id = obj_id.into();
631    let count = Object::find_by_id(obj_id).count(db).await?;
632    if count == 0 {
633        return Err(MetaError::catalog_id_not_found(
634            object_type.as_str(),
635            obj_id,
636        ));
637    }
638    Ok(())
639}
640
641pub async fn ensure_job_not_canceled<C>(job_id: JobId, db: &C) -> MetaResult<()>
642where
643    C: ConnectionTrait,
644{
645    let count = Object::find_by_id(job_id).count(db).await?;
646    if count == 0 {
647        return Err(MetaError::cancelled(format!(
648            "job {} might be cancelled manually or by recovery",
649            job_id
650        )));
651    }
652    Ok(())
653}
654
655/// `ensure_user_id` ensures the existence of target user in the cluster.
656pub async fn ensure_user_id<C>(user_id: UserId, db: &C) -> MetaResult<()>
657where
658    C: ConnectionTrait,
659{
660    let count = User::find_by_id(user_id).count(db).await?;
661    if count == 0 {
662        return Err(anyhow!("user {} was concurrently dropped", user_id).into());
663    }
664    Ok(())
665}
666
667/// `check_database_name_duplicate` checks whether the database name is already used in the cluster.
668pub async fn check_database_name_duplicate<C>(name: &str, db: &C) -> MetaResult<()>
669where
670    C: ConnectionTrait,
671{
672    let count = Database::find()
673        .filter(database::Column::Name.eq(name))
674        .count(db)
675        .await?;
676    if count > 0 {
677        assert_eq!(count, 1);
678        return Err(MetaError::catalog_duplicated("database", name));
679    }
680    Ok(())
681}
682
683/// `check_function_signature_duplicate` checks whether the function name and its signature is already used in the target namespace.
684pub async fn check_function_signature_duplicate<C>(
685    pb_function: &PbFunction,
686    db: &C,
687) -> MetaResult<()>
688where
689    C: ConnectionTrait,
690{
691    let count = Function::find()
692        .inner_join(Object)
693        .filter(
694            object::Column::DatabaseId
695                .eq(pb_function.database_id)
696                .and(object::Column::SchemaId.eq(pb_function.schema_id))
697                .and(function::Column::Name.eq(&pb_function.name))
698                .and(
699                    function::Column::ArgTypes
700                        .eq(DataTypeArray::from(pb_function.arg_types.clone())),
701                ),
702        )
703        .count(db)
704        .await?;
705    if count > 0 {
706        assert_eq!(count, 1);
707        return Err(MetaError::catalog_duplicated("function", &pb_function.name));
708    }
709    Ok(())
710}
711
712/// `check_connection_name_duplicate` checks whether the connection name is already used in the target namespace.
713pub async fn check_connection_name_duplicate<C>(
714    pb_connection: &PbConnection,
715    db: &C,
716) -> MetaResult<()>
717where
718    C: ConnectionTrait,
719{
720    let count = Connection::find()
721        .inner_join(Object)
722        .filter(
723            object::Column::DatabaseId
724                .eq(pb_connection.database_id)
725                .and(object::Column::SchemaId.eq(pb_connection.schema_id))
726                .and(connection::Column::Name.eq(&pb_connection.name)),
727        )
728        .count(db)
729        .await?;
730    if count > 0 {
731        assert_eq!(count, 1);
732        return Err(MetaError::catalog_duplicated(
733            "connection",
734            &pb_connection.name,
735        ));
736    }
737    Ok(())
738}
739
740pub async fn check_secret_name_duplicate<C>(pb_secret: &PbSecret, db: &C) -> MetaResult<()>
741where
742    C: ConnectionTrait,
743{
744    let count = Secret::find()
745        .inner_join(Object)
746        .filter(
747            object::Column::DatabaseId
748                .eq(pb_secret.database_id)
749                .and(object::Column::SchemaId.eq(pb_secret.schema_id))
750                .and(secret::Column::Name.eq(&pb_secret.name)),
751        )
752        .count(db)
753        .await?;
754    if count > 0 {
755        assert_eq!(count, 1);
756        return Err(MetaError::catalog_duplicated("secret", &pb_secret.name));
757    }
758    Ok(())
759}
760
761pub async fn check_subscription_name_duplicate<C>(
762    pb_subscription: &PbSubscription,
763    db: &C,
764) -> MetaResult<()>
765where
766    C: ConnectionTrait,
767{
768    let count = Subscription::find()
769        .inner_join(Object)
770        .filter(
771            object::Column::DatabaseId
772                .eq(pb_subscription.database_id)
773                .and(object::Column::SchemaId.eq(pb_subscription.schema_id))
774                .and(subscription::Column::Name.eq(&pb_subscription.name)),
775        )
776        .count(db)
777        .await?;
778    if count > 0 {
779        assert_eq!(count, 1);
780        return Err(MetaError::catalog_duplicated(
781            "subscription",
782            &pb_subscription.name,
783        ));
784    }
785    Ok(())
786}
787
788/// `check_user_name_duplicate` checks whether the user is already existed in the cluster.
789pub async fn check_user_name_duplicate<C>(name: &str, db: &C) -> MetaResult<()>
790where
791    C: ConnectionTrait,
792{
793    let count = User::find()
794        .filter(user::Column::Name.eq(name))
795        .count(db)
796        .await?;
797    if count > 0 {
798        assert_eq!(count, 1);
799        return Err(MetaError::catalog_duplicated("user", name));
800    }
801    Ok(())
802}
803
804/// `check_relation_name_duplicate` checks whether the relation name is already used in the target namespace.
805pub async fn check_relation_name_duplicate<C>(
806    name: &str,
807    database_id: DatabaseId,
808    schema_id: SchemaId,
809    db: &C,
810) -> MetaResult<()>
811where
812    C: ConnectionTrait,
813{
814    macro_rules! check_duplicated {
815        ($obj_type:expr, $entity:ident, $table:ident) => {
816            let object_id = Object::find()
817                .select_only()
818                .column(object::Column::Oid)
819                .inner_join($entity)
820                .filter(
821                    object::Column::DatabaseId
822                        .eq(Some(database_id))
823                        .and(object::Column::SchemaId.eq(Some(schema_id)))
824                        .and($table::Column::Name.eq(name)),
825                )
826                .into_tuple::<ObjectId>()
827                .one(db)
828                .await?;
829            if let Some(oid) = object_id {
830                let check_creation = if $obj_type == ObjectType::View {
831                    false
832                } else if $obj_type == ObjectType::Source {
833                    let source_info = Source::find_by_id(oid.as_source_id())
834                        .select_only()
835                        .column(source::Column::SourceInfo)
836                        .into_tuple::<Option<StreamSourceInfo>>()
837                        .one(db)
838                        .await?
839                        .unwrap();
840                    source_info.map_or(false, |info| info.to_protobuf().is_shared())
841                } else {
842                    true
843                };
844                let job_id = oid.as_job_id();
845                return if check_creation
846                    && !matches!(
847                        StreamingJob::find_by_id(job_id)
848                            .select_only()
849                            .column(streaming_job::Column::JobStatus)
850                            .into_tuple::<JobStatus>()
851                            .one(db)
852                            .await?,
853                        Some(JobStatus::Created)
854                    ) {
855                    Err(MetaError::catalog_under_creation(
856                        $obj_type.as_str(),
857                        name,
858                        job_id,
859                    ))
860                } else {
861                    Err(MetaError::catalog_duplicated($obj_type.as_str(), name))
862                };
863            }
864        };
865    }
866    check_duplicated!(ObjectType::Table, Table, table);
867    check_duplicated!(ObjectType::Source, Source, source);
868    check_duplicated!(ObjectType::Sink, Sink, sink);
869    check_duplicated!(ObjectType::Index, Index, index);
870    check_duplicated!(ObjectType::View, View, view);
871
872    Ok(())
873}
874
875/// `check_schema_name_duplicate` checks whether the schema name is already used in the target database.
876pub async fn check_schema_name_duplicate<C>(
877    name: &str,
878    database_id: DatabaseId,
879    db: &C,
880) -> MetaResult<()>
881where
882    C: ConnectionTrait,
883{
884    let count = Object::find()
885        .inner_join(Schema)
886        .filter(
887            object::Column::ObjType
888                .eq(ObjectType::Schema)
889                .and(object::Column::DatabaseId.eq(Some(database_id)))
890                .and(schema::Column::Name.eq(name)),
891        )
892        .count(db)
893        .await?;
894    if count != 0 {
895        return Err(MetaError::catalog_duplicated("schema", name));
896    }
897
898    Ok(())
899}
900
901/// Validates a RESTRICT drop and returns referring objects that should be dropped together.
902///
903/// Objects related through `belong_to_oid` are catalog-owned by the target. Indexes are also
904/// logically owned by their primary table for dropping, although they remain schema-owned catalog
905/// objects. These objects are dropped together with the target; all other referrers prevent the
906/// drop.
907pub async fn validate_restrict_drop_and_collect_owned_objects<C>(
908    object_type: ObjectType,
909    object_id: ObjectId,
910    db: &C,
911) -> MetaResult<Vec<PartialObject>>
912where
913    C: ConnectionTrait,
914{
915    let referring_objects = get_referring_objects(object_id, object_type, db).await?;
916    let owned_object_ids = get_belong_objects(db, object_id)
917        .await?
918        .into_iter()
919        .map(|object| object.oid)
920        .collect::<HashSet<_>>();
921    let mut non_owned_objects = referring_objects.clone();
922    non_owned_objects.retain(|object| !owned_object_ids.contains(&object.oid));
923    if object_type == ObjectType::Table {
924        non_owned_objects.retain(|object| object.obj_type != ObjectType::Index);
925    }
926
927    if !non_owned_objects.is_empty() {
928        let referring_objs_map = non_owned_objects
929            .into_iter()
930            .into_group_map_by(|o| o.obj_type);
931        let mut details = vec![];
932        for (obj_type, objs) in referring_objs_map {
933            match obj_type {
934                ObjectType::Table => {
935                    let tables: Vec<(String, String)> = Object::find()
936                        .join(JoinType::InnerJoin, object::Relation::Table.def())
937                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
938                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
939                        .select_only()
940                        .column(schema::Column::Name)
941                        .column(table::Column::Name)
942                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
943                        .into_tuple()
944                        .all(db)
945                        .await?;
946                    details.extend(tables.into_iter().map(|(schema_name, table_name)| {
947                        format!(
948                            "materialized view {}.{} depends on it",
949                            schema_name, table_name
950                        )
951                    }));
952                }
953                ObjectType::Sink => {
954                    let sinks: Vec<(String, String)> = Object::find()
955                        .join(JoinType::InnerJoin, object::Relation::Sink.def())
956                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
957                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
958                        .select_only()
959                        .column(schema::Column::Name)
960                        .column(sink::Column::Name)
961                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
962                        .into_tuple()
963                        .all(db)
964                        .await?;
965                    details.extend(sinks.into_iter().map(|(schema_name, sink_name)| {
966                        format!("sink {}.{} depends on it", schema_name, sink_name)
967                    }));
968                }
969                ObjectType::View => {
970                    let views: Vec<(String, String)> = Object::find()
971                        .join(JoinType::InnerJoin, object::Relation::View.def())
972                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
973                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
974                        .select_only()
975                        .column(schema::Column::Name)
976                        .column(view::Column::Name)
977                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
978                        .into_tuple()
979                        .all(db)
980                        .await?;
981                    details.extend(views.into_iter().map(|(schema_name, view_name)| {
982                        format!("view {}.{} depends on it", schema_name, view_name)
983                    }));
984                }
985                ObjectType::Subscription => {
986                    let subscriptions: Vec<(String, String)> = Object::find()
987                        .join(JoinType::InnerJoin, object::Relation::Subscription.def())
988                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
989                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
990                        .select_only()
991                        .column(schema::Column::Name)
992                        .column(subscription::Column::Name)
993                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
994                        .into_tuple()
995                        .all(db)
996                        .await?;
997                    details.extend(subscriptions.into_iter().map(
998                        |(schema_name, subscription_name)| {
999                            format!(
1000                                "subscription {}.{} depends on it",
1001                                schema_name, subscription_name
1002                            )
1003                        },
1004                    ));
1005                }
1006                ObjectType::Source => {
1007                    let sources: Vec<(String, String)> = Object::find()
1008                        .join(JoinType::InnerJoin, object::Relation::Source.def())
1009                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
1010                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
1011                        .select_only()
1012                        .column(schema::Column::Name)
1013                        .column(source::Column::Name)
1014                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
1015                        .into_tuple()
1016                        .all(db)
1017                        .await?;
1018                    details.extend(sources.into_iter().map(|(schema_name, view_name)| {
1019                        format!("source {}.{} depends on it", schema_name, view_name)
1020                    }));
1021                }
1022                ObjectType::Connection => {
1023                    let connections: Vec<(String, String)> = Object::find()
1024                        .join(JoinType::InnerJoin, object::Relation::Connection.def())
1025                        .join(JoinType::InnerJoin, object::Relation::Database2.def())
1026                        .join(JoinType::InnerJoin, object::Relation::Schema2.def())
1027                        .select_only()
1028                        .column(schema::Column::Name)
1029                        .column(connection::Column::Name)
1030                        .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
1031                        .into_tuple()
1032                        .all(db)
1033                        .await?;
1034                    details.extend(connections.into_iter().map(|(schema_name, view_name)| {
1035                        format!("connection {}.{} depends on it", schema_name, view_name)
1036                    }));
1037                }
1038                // only the table, source, sink, subscription, view, connection and index will depend on other objects.
1039                _ => bail!("unexpected referring object type: {}", obj_type.as_str()),
1040            }
1041        }
1042        if details.is_empty() {
1043            return Ok(referring_objects);
1044        }
1045
1046        return Err(MetaError::permission_denied(format!(
1047            "{} used by {} other objects. \nDETAIL: {}\n\
1048            {}",
1049            object_type.as_str(),
1050            details.len(),
1051            details.join("\n"),
1052            match object_type {
1053                ObjectType::Function | ObjectType::Connection | ObjectType::Secret =>
1054                    "HINT: DROP the dependent objects first.",
1055                ObjectType::Database | ObjectType::Schema => unreachable!(),
1056                _ => "HINT:  Use DROP ... CASCADE to drop the dependent objects too.",
1057            }
1058        )));
1059    }
1060    Ok(referring_objects)
1061}
1062
1063async fn get_incoming_sink_objects_for_target<C>(
1064    target_id: ObjectId,
1065    target_type: ObjectType,
1066    db: &C,
1067) -> MetaResult<Vec<PartialObject>>
1068where
1069    C: ConnectionTrait,
1070{
1071    match target_type {
1072        ObjectType::Table => {
1073            let incoming_sink_ids = Sink::find()
1074                .select_only()
1075                .column(sink::Column::SinkId)
1076                .filter(sink::Column::TargetTable.eq(target_id.as_table_id()))
1077                .into_tuple::<SinkId>()
1078                .all(db)
1079                .await?;
1080            Object::find()
1081                .filter(object::Column::Oid.is_in(incoming_sink_ids))
1082                .into_partial_model()
1083                .all(db)
1084                .await
1085                .map_err(Into::into)
1086        }
1087        _ => Ok(vec![]),
1088    }
1089}
1090
1091/// List all objects that are using the given one.
1092pub async fn get_referring_objects<C>(
1093    object_id: ObjectId,
1094    object_type: ObjectType,
1095    db: &C,
1096) -> MetaResult<Vec<PartialObject>>
1097where
1098    C: ConnectionTrait,
1099{
1100    let mut objects: Vec<PartialObject> = ObjectDependency::find()
1101        .filter(object_dependency::Column::Oid.eq(object_id))
1102        .join(
1103            JoinType::InnerJoin,
1104            object_dependency::Relation::Object1.def(),
1105        )
1106        .into_partial_model()
1107        .all(db)
1108        .await?;
1109
1110    let incoming_sink_objects =
1111        get_incoming_sink_objects_for_target(object_id, object_type, db).await?;
1112    let mut existing_object_ids: HashSet<ObjectId> =
1113        objects.iter().map(|object| object.oid).collect();
1114    objects.extend(
1115        incoming_sink_objects
1116            .into_iter()
1117            .filter(|object| existing_object_ids.insert(object.oid)),
1118    );
1119
1120    Ok(objects)
1121}
1122
1123/// `ensure_schema_empty` ensures that the schema is empty, used by `DROP SCHEMA`.
1124pub async fn ensure_schema_empty<C>(schema_id: SchemaId, db: &C) -> MetaResult<()>
1125where
1126    C: ConnectionTrait,
1127{
1128    let count = Object::find()
1129        .filter(object::Column::SchemaId.eq(Some(schema_id)))
1130        .count(db)
1131        .await?;
1132    if count != 0 {
1133        return Err(MetaError::permission_denied("schema is not empty"));
1134    }
1135
1136    Ok(())
1137}
1138
1139/// `list_user_info_by_ids` lists all users' info by their ids.
1140pub async fn list_user_info_by_ids<C>(
1141    user_ids: impl IntoIterator<Item = UserId>,
1142    db: &C,
1143) -> MetaResult<Vec<PbUserInfo>>
1144where
1145    C: ConnectionTrait,
1146{
1147    let mut user_infos = vec![];
1148    for user_id in user_ids {
1149        let user = User::find_by_id(user_id)
1150            .one(db)
1151            .await?
1152            .ok_or_else(|| MetaError::catalog_id_not_found("user", user_id))?;
1153        let mut user_info: PbUserInfo = user.into();
1154        user_info.grant_privileges = get_user_privilege(user_id, db).await?;
1155        user_infos.push(user_info);
1156    }
1157    Ok(user_infos)
1158}
1159
1160/// `get_object_owner` returns the owner of the given object.
1161pub async fn get_object_owner<C>(object_id: ObjectId, db: &C) -> MetaResult<UserId>
1162where
1163    C: ConnectionTrait,
1164{
1165    let obj_owner: UserId = Object::find_by_id(object_id)
1166        .select_only()
1167        .column(object::Column::OwnerId)
1168        .into_tuple()
1169        .one(db)
1170        .await?
1171        .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1172    Ok(obj_owner)
1173}
1174
1175/// `construct_privilege_dependency_query` constructs a query to find all privileges that are dependent on the given one.
1176///
1177/// # Examples
1178///
1179/// ```
1180/// use risingwave_meta::controller::utils::construct_privilege_dependency_query;
1181/// use sea_orm::sea_query::*;
1182/// use sea_orm::*;
1183///
1184/// let query = construct_privilege_dependency_query(vec![1, 2, 3]);
1185///
1186/// assert_eq!(
1187///    query.to_string(MysqlQueryBuilder),
1188///   r#"WITH RECURSIVE `granted_privilege_ids` (`id`, `user_id`) AS (SELECT `id`, `user_id` FROM `user_privilege` WHERE `user_privilege`.`id` IN (1, 2, 3) UNION ALL (SELECT `user_privilege`.`id`, `user_privilege`.`user_id` FROM `user_privilege` INNER JOIN `granted_privilege_ids` ON `granted_privilege_ids`.`id` = `dependent_id`)) SELECT `id`, `user_id` FROM `granted_privilege_ids`"#
1189/// );
1190/// assert_eq!(
1191///   query.to_string(PostgresQueryBuilder),
1192///  r#"WITH RECURSIVE "granted_privilege_ids" ("id", "user_id") AS (SELECT "id", "user_id" FROM "user_privilege" WHERE "user_privilege"."id" IN (1, 2, 3) UNION ALL (SELECT "user_privilege"."id", "user_privilege"."user_id" FROM "user_privilege" INNER JOIN "granted_privilege_ids" ON "granted_privilege_ids"."id" = "dependent_id")) SELECT "id", "user_id" FROM "granted_privilege_ids""#
1193/// );
1194/// assert_eq!(
1195///  query.to_string(SqliteQueryBuilder),
1196///  r#"WITH RECURSIVE "granted_privilege_ids" ("id", "user_id") AS (SELECT "id", "user_id" FROM "user_privilege" WHERE "user_privilege"."id" IN (1, 2, 3) UNION ALL SELECT "user_privilege"."id", "user_privilege"."user_id" FROM "user_privilege" INNER JOIN "granted_privilege_ids" ON "granted_privilege_ids"."id" = "dependent_id") SELECT "id", "user_id" FROM "granted_privilege_ids""#
1197/// );
1198/// ```
1199pub fn construct_privilege_dependency_query(ids: Vec<PrivilegeId>) -> WithQuery {
1200    let cte_alias = Alias::new("granted_privilege_ids");
1201    let cte_return_privilege_alias = Alias::new("id");
1202    let cte_return_user_alias = Alias::new("user_id");
1203
1204    let mut base_query = SelectStatement::new()
1205        .columns([user_privilege::Column::Id, user_privilege::Column::UserId])
1206        .from(UserPrivilege)
1207        .and_where(user_privilege::Column::Id.is_in(ids))
1208        .to_owned();
1209
1210    let cte_referencing = Query::select()
1211        .columns([
1212            (UserPrivilege, user_privilege::Column::Id),
1213            (UserPrivilege, user_privilege::Column::UserId),
1214        ])
1215        .from(UserPrivilege)
1216        .inner_join(
1217            cte_alias.clone(),
1218            Expr::col((cte_alias.clone(), cte_return_privilege_alias.clone()))
1219                .equals(user_privilege::Column::DependentId),
1220        )
1221        .to_owned();
1222
1223    let mut common_table_expr = CommonTableExpression::new();
1224    common_table_expr
1225        .query(base_query.union(UnionType::All, cte_referencing).to_owned())
1226        .columns([
1227            cte_return_privilege_alias.clone(),
1228            cte_return_user_alias.clone(),
1229        ])
1230        .table_name(cte_alias.clone());
1231
1232    SelectStatement::new()
1233        .columns([cte_return_privilege_alias, cte_return_user_alias])
1234        .from(cte_alias)
1235        .to_owned()
1236        .with(
1237            WithClause::new()
1238                .recursive(true)
1239                .cte(common_table_expr)
1240                .to_owned(),
1241        )
1242}
1243
1244pub async fn get_internal_tables_by_id<C>(job_id: JobId, db: &C) -> MetaResult<Vec<TableId>>
1245where
1246    C: ConnectionTrait,
1247{
1248    let table_ids: Vec<TableId> = Table::find()
1249        .select_only()
1250        .column(table::Column::TableId)
1251        .filter(
1252            table::Column::TableType
1253                .eq(TableType::Internal)
1254                .and(table::Column::BelongsToJobId.eq(job_id)),
1255        )
1256        .into_tuple()
1257        .all(db)
1258        .await?;
1259    Ok(table_ids)
1260}
1261
1262pub async fn get_index_state_tables_by_table_id<C>(
1263    table_id: TableId,
1264    db: &C,
1265) -> MetaResult<Vec<TableId>>
1266where
1267    C: ConnectionTrait,
1268{
1269    let mut index_table_ids: Vec<TableId> = Index::find()
1270        .select_only()
1271        .column(index::Column::IndexTableId)
1272        .filter(index::Column::PrimaryTableId.eq(table_id))
1273        .into_tuple()
1274        .all(db)
1275        .await?;
1276
1277    if !index_table_ids.is_empty() {
1278        let internal_table_ids: Vec<TableId> = Table::find()
1279            .select_only()
1280            .column(table::Column::TableId)
1281            .filter(
1282                table::Column::TableType
1283                    .eq(TableType::Internal)
1284                    .and(table::Column::BelongsToJobId.is_in(index_table_ids.clone())),
1285            )
1286            .into_tuple()
1287            .all(db)
1288            .await?;
1289
1290        index_table_ids.extend(internal_table_ids);
1291    }
1292
1293    Ok(index_table_ids)
1294}
1295
1296/// `get_iceberg_related_object_ids` returns the related object ids of the iceberg source, sink and internal tables.
1297pub async fn get_iceberg_related_object_ids<C>(
1298    object_id: ObjectId,
1299    db: &C,
1300) -> MetaResult<Vec<ObjectId>>
1301where
1302    C: ConnectionTrait,
1303{
1304    let object = Object::find_by_id(object_id)
1305        .one(db)
1306        .await?
1307        .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1308    if object.obj_type != ObjectType::Table {
1309        return Ok(vec![]);
1310    }
1311
1312    let table = Table::find_by_id(object_id.as_table_id())
1313        .one(db)
1314        .await?
1315        .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
1316    if !matches!(table.engine, Some(table::Engine::Iceberg)) {
1317        return Ok(vec![]);
1318    }
1319
1320    let database_id = object.database_id.unwrap();
1321    let schema_id = object.schema_id.unwrap();
1322
1323    let mut related_objects = vec![];
1324
1325    let iceberg_sink_name = format!("{}{}", ICEBERG_SINK_PREFIX, table.name);
1326    let iceberg_sink_id = Sink::find()
1327        .inner_join(Object)
1328        .select_only()
1329        .column(sink::Column::SinkId)
1330        .filter(
1331            object::Column::DatabaseId
1332                .eq(database_id)
1333                .and(object::Column::SchemaId.eq(schema_id))
1334                .and(sink::Column::Name.eq(&iceberg_sink_name)),
1335        )
1336        .into_tuple::<SinkId>()
1337        .one(db)
1338        .await?;
1339    if let Some(sink_id) = iceberg_sink_id {
1340        related_objects.push(sink_id.as_object_id());
1341        let sink_internal_tables = get_internal_tables_by_id(sink_id.as_job_id(), db).await?;
1342        related_objects.extend(
1343            sink_internal_tables
1344                .into_iter()
1345                .map(|tid| tid.as_object_id()),
1346        );
1347    } else {
1348        warn!(
1349            "iceberg table {} missing sink {}",
1350            table.name, iceberg_sink_name
1351        );
1352    }
1353
1354    let iceberg_source_name = format!("{}{}", ICEBERG_SOURCE_PREFIX, table.name);
1355    let iceberg_source_id = Source::find()
1356        .inner_join(Object)
1357        .select_only()
1358        .column(source::Column::SourceId)
1359        .filter(
1360            object::Column::DatabaseId
1361                .eq(database_id)
1362                .and(object::Column::SchemaId.eq(schema_id))
1363                .and(source::Column::Name.eq(&iceberg_source_name)),
1364        )
1365        .into_tuple::<SourceId>()
1366        .one(db)
1367        .await?;
1368    if let Some(source_id) = iceberg_source_id {
1369        related_objects.push(source_id.as_object_id());
1370    } else {
1371        warn!(
1372            "iceberg table {} missing source {}",
1373            table.name, iceberg_source_name
1374        );
1375    }
1376
1377    Ok(related_objects)
1378}
1379
1380/// Load streaming jobs by job ids.
1381pub(crate) async fn load_streaming_jobs_by_ids<C>(
1382    txn: &C,
1383    job_ids: impl IntoIterator<Item = JobId>,
1384) -> MetaResult<HashMap<JobId, streaming_job::Model>>
1385where
1386    C: ConnectionTrait,
1387{
1388    let job_ids: HashSet<JobId> = job_ids.into_iter().collect();
1389    if job_ids.is_empty() {
1390        return Ok(HashMap::new());
1391    }
1392    let jobs = streaming_job::Entity::find()
1393        .filter(streaming_job::Column::JobId.is_in(job_ids.clone()))
1394        .all(txn)
1395        .await?;
1396    Ok(jobs.into_iter().map(|job| (job.job_id, job)).collect())
1397}
1398
1399#[derive(Clone, DerivePartialModel, FromQueryResult)]
1400#[sea_orm(entity = "UserPrivilege")]
1401pub struct PartialUserPrivilege {
1402    pub id: PrivilegeId,
1403    pub user_id: UserId,
1404}
1405
1406pub async fn get_referring_privileges_cascade<C>(
1407    ids: Vec<PrivilegeId>,
1408    db: &C,
1409) -> MetaResult<Vec<PartialUserPrivilege>>
1410where
1411    C: ConnectionTrait,
1412{
1413    let query = construct_privilege_dependency_query(ids);
1414    let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
1415    let privileges = PartialUserPrivilege::find_by_statement(Statement::from_sql_and_values(
1416        db.get_database_backend(),
1417        sql,
1418        values,
1419    ))
1420    .all(db)
1421    .await?;
1422
1423    Ok(privileges)
1424}
1425
1426/// `ensure_privileges_not_referred` ensures that the privileges are not granted to any other users.
1427pub async fn ensure_privileges_not_referred<C>(ids: Vec<PrivilegeId>, db: &C) -> MetaResult<()>
1428where
1429    C: ConnectionTrait,
1430{
1431    let count = UserPrivilege::find()
1432        .filter(user_privilege::Column::DependentId.is_in(ids))
1433        .count(db)
1434        .await?;
1435    if count != 0 {
1436        return Err(MetaError::permission_denied(format!(
1437            "privileges granted to {} other ones.",
1438            count
1439        )));
1440    }
1441    Ok(())
1442}
1443
1444/// `get_user_privilege` returns the privileges of the given user.
1445pub async fn get_user_privilege<C>(user_id: UserId, db: &C) -> MetaResult<Vec<PbGrantPrivilege>>
1446where
1447    C: ConnectionTrait,
1448{
1449    let user_privileges = UserPrivilege::find()
1450        .find_also_related(Object)
1451        .filter(user_privilege::Column::UserId.eq(user_id))
1452        .all(db)
1453        .await?;
1454    Ok(user_privileges
1455        .into_iter()
1456        .map(|(privilege, object)| {
1457            let object = object.unwrap();
1458            let obj = match object.obj_type {
1459                ObjectType::Database => PbGrantObject::DatabaseId(object.oid.as_database_id()),
1460                ObjectType::Schema => PbGrantObject::SchemaId(object.oid.as_schema_id()),
1461                ObjectType::Table | ObjectType::Index => {
1462                    PbGrantObject::TableId(object.oid.as_table_id())
1463                }
1464                ObjectType::Source => PbGrantObject::SourceId(object.oid.as_source_id()),
1465                ObjectType::Sink => PbGrantObject::SinkId(object.oid.as_sink_id()),
1466                ObjectType::View => PbGrantObject::ViewId(object.oid.as_view_id()),
1467                ObjectType::Function => PbGrantObject::FunctionId(object.oid.as_function_id()),
1468                ObjectType::Connection => {
1469                    PbGrantObject::ConnectionId(object.oid.as_connection_id())
1470                }
1471                ObjectType::Subscription => {
1472                    PbGrantObject::SubscriptionId(object.oid.as_subscription_id())
1473                }
1474                ObjectType::Secret => PbGrantObject::SecretId(object.oid.as_secret_id()),
1475            };
1476            PbGrantPrivilege {
1477                action_with_opts: vec![PbActionWithGrantOption {
1478                    action: PbAction::from(privilege.action) as _,
1479                    with_grant_option: privilege.with_grant_option,
1480                    granted_by: privilege.granted_by as _,
1481                }],
1482                object: Some(obj),
1483            }
1484        })
1485        .collect())
1486}
1487
1488/// Insert user privileges idempotently using the same key as `idx_user_privilege_item`.
1489/// An existing privilege is preserved, except that its grant option is upgraded when requested.
1490pub(crate) async fn upsert_user_privileges<C>(
1491    db: &C,
1492    privileges: impl IntoIterator<Item = user_privilege::ActiveModel>,
1493) -> MetaResult<()>
1494where
1495    C: ConnectionTrait,
1496{
1497    for privilege in privileges {
1498        let mut on_conflict = OnConflict::columns([
1499            user_privilege::Column::UserId,
1500            user_privilege::Column::Oid,
1501            user_privilege::Column::Action,
1502            user_privilege::Column::GrantedBy,
1503        ]);
1504        if *privilege.with_grant_option.as_ref() {
1505            on_conflict.update_column(user_privilege::Column::WithGrantOption);
1506        } else {
1507            // Workaround to support MYSQL for `DO NOTHING`.
1508            on_conflict.update_column(user_privilege::Column::UserId);
1509        }
1510
1511        UserPrivilege::insert(privilege)
1512            .on_conflict(on_conflict)
1513            .do_nothing()
1514            .exec(db)
1515            .await?;
1516    }
1517    Ok(())
1518}
1519
1520pub async fn get_table_columns(
1521    txn: &impl ConnectionTrait,
1522    id: TableId,
1523) -> MetaResult<ColumnCatalogArray> {
1524    let columns = Table::find_by_id(id)
1525        .select_only()
1526        .columns([table::Column::Columns])
1527        .into_tuple::<ColumnCatalogArray>()
1528        .one(txn)
1529        .await?
1530        .ok_or_else(|| MetaError::catalog_id_not_found("table", id))?;
1531    Ok(columns)
1532}
1533
1534/// `grant_default_privileges_automatically` grants default privileges automatically
1535/// for the given new object. It returns the list of user infos whose privileges are updated.
1536pub async fn grant_default_privileges_automatically<C>(
1537    db: &C,
1538    object_id: impl Into<ObjectId>,
1539) -> MetaResult<Vec<PbUserInfo>>
1540where
1541    C: ConnectionTrait,
1542{
1543    let object_id = object_id.into();
1544    let object = Object::find_by_id(object_id)
1545        .one(db)
1546        .await?
1547        .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1548    assert_ne!(object.obj_type, ObjectType::Database);
1549
1550    let for_mview_filter = if object.obj_type == ObjectType::Table {
1551        let table_type = Table::find_by_id(object_id.as_table_id())
1552            .select_only()
1553            .column(table::Column::TableType)
1554            .into_tuple::<TableType>()
1555            .one(db)
1556            .await?
1557            .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
1558        user_default_privilege::Column::ForMaterializedView
1559            .eq(table_type == TableType::MaterializedView)
1560    } else {
1561        user_default_privilege::Column::ForMaterializedView.eq(false)
1562    };
1563    let schema_filter = if let Some(schema_id) = &object.schema_id {
1564        user_default_privilege::Column::SchemaId.eq(*schema_id)
1565    } else {
1566        user_default_privilege::Column::SchemaId.is_null()
1567    };
1568
1569    let default_privileges: Vec<(UserId, UserId, Action, bool)> = UserDefaultPrivilege::find()
1570        .select_only()
1571        .columns([
1572            user_default_privilege::Column::Grantee,
1573            user_default_privilege::Column::GrantedBy,
1574            user_default_privilege::Column::Action,
1575            user_default_privilege::Column::WithGrantOption,
1576        ])
1577        .filter(
1578            user_default_privilege::Column::DatabaseId
1579                .eq(object.database_id.unwrap())
1580                .and(schema_filter)
1581                .and(user_default_privilege::Column::UserId.eq(object.owner_id))
1582                .and(user_default_privilege::Column::ObjectType.eq(object.obj_type))
1583                .and(for_mview_filter),
1584        )
1585        .into_tuple()
1586        .all(db)
1587        .await?;
1588    if default_privileges.is_empty() {
1589        return Ok(vec![]);
1590    }
1591
1592    let updated_user_ids = default_privileges
1593        .iter()
1594        .map(|(grantee, _, _, _)| *grantee)
1595        .collect::<HashSet<_>>();
1596
1597    let mut new_privileges = vec![];
1598    for (grantee, granted_by, action, with_grant_option) in default_privileges {
1599        new_privileges.push(user_privilege::ActiveModel {
1600            user_id: Set(grantee),
1601            oid: Set(object_id),
1602            granted_by: Set(granted_by),
1603            action: Set(action),
1604            with_grant_option: Set(with_grant_option),
1605            ..Default::default()
1606        });
1607        if action == Action::Select {
1608            // Grant SELECT privilege for internal tables if the action is SELECT.
1609            let internal_table_ids = get_internal_tables_by_id(object_id.as_job_id(), db).await?;
1610            new_privileges.extend(internal_table_ids.into_iter().map(|internal_table_id| {
1611                user_privilege::ActiveModel {
1612                    user_id: Set(grantee),
1613                    oid: Set(internal_table_id.as_object_id()),
1614                    granted_by: Set(granted_by),
1615                    action: Set(Action::Select),
1616                    with_grant_option: Set(with_grant_option),
1617                    ..Default::default()
1618                }
1619            }));
1620
1621            // Additionally, grant SELECT privilege for iceberg related objects if the action is SELECT.
1622            let iceberg_privilege_object_ids =
1623                get_iceberg_related_object_ids(object_id, db).await?;
1624            new_privileges.extend(iceberg_privilege_object_ids.into_iter().map(
1625                |iceberg_object_id| user_privilege::ActiveModel {
1626                    user_id: Set(grantee),
1627                    oid: Set(iceberg_object_id),
1628                    granted_by: Set(granted_by),
1629                    action: Set(action),
1630                    with_grant_option: Set(with_grant_option),
1631                    ..Default::default()
1632                },
1633            ));
1634        }
1635    }
1636    upsert_user_privileges(db, new_privileges).await?;
1637
1638    let updated_user_infos = list_user_info_by_ids(updated_user_ids, db).await?;
1639    Ok(updated_user_infos)
1640}
1641
1642// todo: remove it after migrated to sql backend.
1643pub fn extract_grant_obj_id(object: &PbGrantObject) -> ObjectId {
1644    match object {
1645        PbGrantObject::DatabaseId(id) => (*id).into(),
1646        PbGrantObject::SchemaId(id) => (*id).into(),
1647        PbGrantObject::TableId(id) => (*id).into(),
1648        PbGrantObject::SourceId(id) => (*id).into(),
1649        PbGrantObject::SinkId(id) => (*id).into(),
1650        PbGrantObject::ViewId(id) => (*id).into(),
1651        PbGrantObject::FunctionId(id) => (*id).into(),
1652        PbGrantObject::SubscriptionId(id) => (*id).into(),
1653        PbGrantObject::ConnectionId(id) => (*id).into(),
1654        PbGrantObject::SecretId(id) => (*id).into(),
1655    }
1656}
1657
1658pub async fn insert_fragment_relations(
1659    db: &impl ConnectionTrait,
1660    downstream_fragment_relations: &FragmentDownstreamRelation,
1661) -> MetaResult<()> {
1662    let mut relations = vec![];
1663    for (upstream_fragment_id, downstreams) in downstream_fragment_relations {
1664        for downstream in downstreams {
1665            relations.push(
1666                fragment_relation::Model {
1667                    source_fragment_id: *upstream_fragment_id as _,
1668                    target_fragment_id: downstream.downstream_fragment_id as _,
1669                    dispatcher_type: downstream.dispatcher_type,
1670                    dist_key_indices: downstream
1671                        .dist_key_indices
1672                        .iter()
1673                        .map(|idx| *idx as i32)
1674                        .collect_vec()
1675                        .into(),
1676                    output_indices: downstream
1677                        .output_mapping
1678                        .indices
1679                        .iter()
1680                        .map(|idx| *idx as i32)
1681                        .collect_vec()
1682                        .into(),
1683                    output_type_mapping: Some(downstream.output_mapping.types.clone().into()),
1684                }
1685                .into_active_model(),
1686            );
1687        }
1688    }
1689    if !relations.is_empty() {
1690        FragmentRelation::insert_many(relations).exec(db).await?;
1691    }
1692    Ok(())
1693}
1694
1695pub fn compose_dispatchers(
1696    source_fragment_distribution: DistributionType,
1697    source_fragment_actors: &HashMap<crate::model::ActorId, Option<Bitmap>>,
1698    target_fragment_id: crate::model::FragmentId,
1699    target_fragment_distribution: DistributionType,
1700    target_fragment_actors: &HashMap<crate::model::ActorId, Option<Bitmap>>,
1701    dispatcher_type: DispatcherType,
1702    dist_key_indices: Vec<u32>,
1703    output_mapping: PbDispatchOutputMapping,
1704) -> (
1705    HashMap<crate::model::ActorId, PbDispatcher>,
1706    Option<HashMap<crate::model::ActorId, crate::model::ActorId>>,
1707) {
1708    match dispatcher_type {
1709        DispatcherType::Hash => {
1710            let dispatcher = PbDispatcher {
1711                r#type: PbDispatcherType::from(dispatcher_type) as _,
1712                dist_key_indices,
1713                output_mapping: output_mapping.into(),
1714                hash_mapping: Some(
1715                    ActorMapping::from_bitmaps(
1716                        &target_fragment_actors
1717                            .iter()
1718                            .map(|(actor_id, bitmap)| {
1719                                (
1720                                    *actor_id as _,
1721                                    bitmap
1722                                        .clone()
1723                                        .expect("downstream hash dispatch must have distribution"),
1724                                )
1725                            })
1726                            .collect(),
1727                    )
1728                    .to_protobuf(),
1729                ),
1730                dispatcher_id: target_fragment_id,
1731                downstream_actor_id: target_fragment_actors.keys().copied().collect(),
1732            };
1733            (
1734                source_fragment_actors
1735                    .keys()
1736                    .map(|source_actor_id| (*source_actor_id, dispatcher.clone()))
1737                    .collect(),
1738                None,
1739            )
1740        }
1741        DispatcherType::Broadcast | DispatcherType::Simple => {
1742            let dispatcher = PbDispatcher {
1743                r#type: PbDispatcherType::from(dispatcher_type) as _,
1744                dist_key_indices,
1745                output_mapping: output_mapping.into(),
1746                hash_mapping: None,
1747                dispatcher_id: target_fragment_id,
1748                downstream_actor_id: target_fragment_actors.keys().copied().collect(),
1749            };
1750            (
1751                source_fragment_actors
1752                    .keys()
1753                    .map(|source_actor_id| (*source_actor_id, dispatcher.clone()))
1754                    .collect(),
1755                None,
1756            )
1757        }
1758        DispatcherType::NoShuffle => {
1759            let no_shuffle_map = resolve_no_shuffle_actor_mapping(
1760                source_fragment_distribution,
1761                source_fragment_actors
1762                    .iter()
1763                    .map(|(&id, bitmap)| (id, bitmap)),
1764                target_fragment_distribution,
1765                target_fragment_actors
1766                    .iter()
1767                    .map(|(&id, bitmap)| (id, bitmap)),
1768            );
1769            let dispatchers = no_shuffle_map
1770                .iter()
1771                .map(|(&upstream_actor_id, &downstream_actor_id)| {
1772                    (
1773                        upstream_actor_id,
1774                        PbDispatcher {
1775                            r#type: PbDispatcherType::NoShuffle as _,
1776                            dist_key_indices: dist_key_indices.clone(),
1777                            output_mapping: output_mapping.clone().into(),
1778                            hash_mapping: None,
1779                            dispatcher_id: target_fragment_id,
1780                            downstream_actor_id: vec![downstream_actor_id],
1781                        },
1782                    )
1783                })
1784                .collect();
1785            (dispatchers, Some(no_shuffle_map))
1786        }
1787    }
1788}
1789
1790/// Resolve 1:1 actor mapping between two no-shuffle-connected fragments.
1791///
1792/// Returns `Vec<(upstream_actor_id, downstream_actor_id)>` pairs. The actor id
1793/// type is generic so the same logic can be reused both for real `ActorId` mapping
1794/// and for alignment checks with synthetic ids (e.g. `(WorkerId, actor_idx)`).
1795///
1796/// For `Single` distribution the single actors are paired directly.
1797/// For `Hash` distribution actors are matched by their bitmap's first vnode and
1798/// full bitmap equality is asserted.
1799pub fn resolve_no_shuffle_actor_mapping<
1800    'a,
1801    ActorId: Copy + Eq + std::hash::Hash + std::fmt::Debug,
1802>(
1803    source_fragment_distribution: DistributionType,
1804    source_fragment_actors: impl IntoIterator<Item = (ActorId, &'a Option<Bitmap>)>,
1805    target_fragment_distribution: DistributionType,
1806    target_fragment_actors: impl IntoIterator<Item = (ActorId, &'a Option<Bitmap>)>,
1807) -> HashMap<ActorId, ActorId> {
1808    assert_eq!(source_fragment_distribution, target_fragment_distribution);
1809
1810    match source_fragment_distribution {
1811        DistributionType::Single => {
1812            let assert_singleton = |bitmap: &Option<Bitmap>| {
1813                assert!(
1814                    bitmap.as_ref().map(|bitmap| bitmap.all()).unwrap_or(true),
1815                    "not singleton: {:?}",
1816                    bitmap
1817                );
1818            };
1819            let (source_actor_id, bitmap) =
1820                Itertools::exactly_one(source_fragment_actors.into_iter())
1821                    .ok()
1822                    .expect("Single distribution should have exactly one source actor");
1823            assert_singleton(bitmap);
1824            let (target_actor_id, bitmap) =
1825                Itertools::exactly_one(target_fragment_actors.into_iter())
1826                    .ok()
1827                    .expect("Single distribution should have exactly one target actor");
1828            assert_singleton(bitmap);
1829            HashMap::from([(source_actor_id, target_actor_id)])
1830        }
1831        DistributionType::Hash => {
1832            // Build target index keyed by first vnode (one pass over target).
1833            let mut target_by_vnode: HashMap<_, _> = target_fragment_actors
1834                .into_iter()
1835                .map(|(actor_id, bitmap)| {
1836                    let bitmap = bitmap
1837                        .as_ref()
1838                        .expect("hash distribution should have bitmap");
1839                    let first_vnode = bitmap.iter_vnodes().next().expect("non-empty bitmap");
1840                    (first_vnode, (actor_id, bitmap))
1841                })
1842                .collect();
1843
1844            let target_count = target_by_vnode.len();
1845
1846            // One pass over source, matching against the target index.
1847            let mapping: HashMap<_, _> = source_fragment_actors
1848                .into_iter()
1849                .map(|(source_actor_id, source_bitmap)| {
1850                    let source_bitmap = source_bitmap
1851                        .as_ref()
1852                        .expect("hash distribution should have bitmap");
1853                    let first_vnode = source_bitmap
1854                        .iter_vnodes()
1855                        .next()
1856                        .expect("non-empty bitmap");
1857                    let (target_actor_id, target_bitmap) =
1858                        target_by_vnode.remove(&first_vnode).unwrap_or_else(|| {
1859                            panic!(
1860                                "cannot find matched target actor: {:?} first_vnode {:?}",
1861                                source_actor_id, first_vnode,
1862                            )
1863                        });
1864                    assert_eq!(
1865                        source_bitmap, target_bitmap,
1866                        "bitmap mismatch for source {:?} target {:?} at first_vnode {:?}",
1867                        source_actor_id, target_actor_id, first_vnode,
1868                    );
1869                    (source_actor_id, target_actor_id)
1870                })
1871                .collect();
1872
1873            assert_eq!(
1874                mapping.len(),
1875                target_count,
1876                "no-shuffle should have equal upstream downstream actor count: {} vs {}",
1877                mapping.len(),
1878                target_count,
1879            );
1880
1881            mapping
1882        }
1883    }
1884}
1885
1886pub fn rebuild_fragment_mapping(fragment: &SharedFragmentInfo) -> PbFragmentWorkerSlotMapping {
1887    let fragment_worker_slot_mapping = match fragment.distribution_type {
1888        DistributionType::Single => {
1889            let actor = Itertools::exactly_one(fragment.actors.values()).unwrap();
1890            WorkerSlotMapping::new_single(WorkerSlotId::new(actor.worker_id as _, 0))
1891        }
1892        DistributionType::Hash => {
1893            let actor_bitmaps: HashMap<_, _> = fragment
1894                .actors
1895                .iter()
1896                .map(|(actor_id, actor_info)| {
1897                    let vnode_bitmap = actor_info
1898                        .vnode_bitmap
1899                        .as_ref()
1900                        .cloned()
1901                        .expect("actor bitmap shouldn't be none in hash fragment");
1902
1903                    (*actor_id as hash::ActorId, vnode_bitmap)
1904                })
1905                .collect();
1906
1907            let actor_mapping = ActorMapping::from_bitmaps(&actor_bitmaps);
1908
1909            let actor_locations = fragment
1910                .actors
1911                .iter()
1912                .map(|(actor_id, actor_info)| (*actor_id as hash::ActorId, actor_info.worker_id))
1913                .collect();
1914
1915            actor_mapping.to_worker_slot(&actor_locations)
1916        }
1917    };
1918
1919    PbFragmentWorkerSlotMapping {
1920        fragment_id: fragment.fragment_id,
1921        mapping: Some(fragment_worker_slot_mapping.to_protobuf()),
1922    }
1923}
1924
1925/// For the given streaming jobs, returns
1926/// - All source fragments
1927/// - All sink fragments
1928/// - All fragments
1929pub async fn get_fragments_for_jobs<C>(
1930    db: &C,
1931    streaming_jobs: Vec<JobId>,
1932) -> MetaResult<(
1933    HashMap<SourceId, BTreeSet<FragmentId>>,
1934    HashSet<FragmentId>,
1935    HashSet<FragmentId>,
1936)>
1937where
1938    C: ConnectionTrait,
1939{
1940    if streaming_jobs.is_empty() {
1941        return Ok((HashMap::default(), HashSet::default(), HashSet::default()));
1942    }
1943
1944    let fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find()
1945        .select_only()
1946        .columns([
1947            fragment::Column::FragmentId,
1948            fragment::Column::FragmentTypeMask,
1949            fragment::Column::StreamNode,
1950        ])
1951        .filter(fragment::Column::JobId.is_in(streaming_jobs))
1952        .into_tuple()
1953        .all(db)
1954        .await?;
1955
1956    let fragment_ids: HashSet<_> = fragments
1957        .iter()
1958        .map(|(fragment_id, _, _)| *fragment_id)
1959        .collect();
1960
1961    let mut source_fragment_ids: HashMap<SourceId, BTreeSet<FragmentId>> = HashMap::new();
1962    let mut sink_fragment_ids: HashSet<FragmentId> = HashSet::new();
1963    for (fragment_id, mask, stream_node) in fragments {
1964        if FragmentTypeMask::from(mask).contains(FragmentTypeFlag::Source)
1965            && let Some(source_id) = stream_node.to_protobuf().find_stream_source()
1966        {
1967            source_fragment_ids
1968                .entry(source_id)
1969                .or_default()
1970                .insert(fragment_id);
1971        }
1972        if FragmentTypeMask::from(mask).contains(FragmentTypeFlag::Sink) {
1973            sink_fragment_ids.insert(fragment_id);
1974        }
1975    }
1976
1977    Ok((source_fragment_ids, sink_fragment_ids, fragment_ids))
1978}
1979
1980/// Build a object group for notifying the deletion of the given objects.
1981///
1982/// Note that only id fields are filled in the object info, as the arguments are partial objects.
1983/// As a result, the returned notification info should only be used for deletion.
1984pub(crate) fn build_object_group_for_delete(
1985    partial_objects: Vec<PartialObject>,
1986) -> NotificationInfo {
1987    let mut objects = vec![];
1988    for obj in partial_objects {
1989        match obj.obj_type {
1990            ObjectType::Database => objects.push(PbObject {
1991                object_info: Some(PbObjectInfo::Database(PbDatabase {
1992                    id: obj.oid.as_database_id(),
1993                    ..Default::default()
1994                })),
1995            }),
1996            ObjectType::Schema => objects.push(PbObject {
1997                object_info: Some(PbObjectInfo::Schema(PbSchema {
1998                    id: obj.oid.as_schema_id(),
1999                    database_id: obj.database_id.unwrap(),
2000                    ..Default::default()
2001                })),
2002            }),
2003            ObjectType::Table => objects.push(PbObject {
2004                object_info: Some(PbObjectInfo::Table(PbTable {
2005                    id: obj.oid.as_table_id(),
2006                    schema_id: obj.schema_id.unwrap(),
2007                    database_id: obj.database_id.unwrap(),
2008                    ..Default::default()
2009                })),
2010            }),
2011            ObjectType::Source => objects.push(PbObject {
2012                object_info: Some(PbObjectInfo::Source(PbSource {
2013                    id: obj.oid.as_source_id(),
2014                    schema_id: obj.schema_id.unwrap(),
2015                    database_id: obj.database_id.unwrap(),
2016                    ..Default::default()
2017                })),
2018            }),
2019            ObjectType::Sink => objects.push(PbObject {
2020                object_info: Some(PbObjectInfo::Sink(PbSink {
2021                    id: obj.oid.as_sink_id(),
2022                    schema_id: obj.schema_id.unwrap(),
2023                    database_id: obj.database_id.unwrap(),
2024                    ..Default::default()
2025                })),
2026            }),
2027            ObjectType::Subscription => objects.push(PbObject {
2028                object_info: Some(PbObjectInfo::Subscription(PbSubscription {
2029                    id: obj.oid.as_subscription_id(),
2030                    schema_id: obj.schema_id.unwrap(),
2031                    database_id: obj.database_id.unwrap(),
2032                    ..Default::default()
2033                })),
2034            }),
2035            ObjectType::View => objects.push(PbObject {
2036                object_info: Some(PbObjectInfo::View(PbView {
2037                    id: obj.oid.as_view_id(),
2038                    schema_id: obj.schema_id.unwrap(),
2039                    database_id: obj.database_id.unwrap(),
2040                    ..Default::default()
2041                })),
2042            }),
2043            ObjectType::Index => {
2044                objects.push(PbObject {
2045                    object_info: Some(PbObjectInfo::Index(PbIndex {
2046                        id: obj.oid.as_index_id(),
2047                        schema_id: obj.schema_id.unwrap(),
2048                        database_id: obj.database_id.unwrap(),
2049                        ..Default::default()
2050                    })),
2051                });
2052                objects.push(PbObject {
2053                    object_info: Some(PbObjectInfo::Table(PbTable {
2054                        id: obj.oid.as_table_id(),
2055                        schema_id: obj.schema_id.unwrap(),
2056                        database_id: obj.database_id.unwrap(),
2057                        ..Default::default()
2058                    })),
2059                });
2060            }
2061            ObjectType::Function => objects.push(PbObject {
2062                object_info: Some(PbObjectInfo::Function(PbFunction {
2063                    id: obj.oid.as_function_id(),
2064                    schema_id: obj.schema_id.unwrap(),
2065                    database_id: obj.database_id.unwrap(),
2066                    ..Default::default()
2067                })),
2068            }),
2069            ObjectType::Connection => objects.push(PbObject {
2070                object_info: Some(PbObjectInfo::Connection(PbConnection {
2071                    id: obj.oid.as_connection_id(),
2072                    schema_id: obj.schema_id.unwrap(),
2073                    database_id: obj.database_id.unwrap(),
2074                    ..Default::default()
2075                })),
2076            }),
2077            ObjectType::Secret => objects.push(PbObject {
2078                object_info: Some(PbObjectInfo::Secret(PbSecret {
2079                    id: obj.oid.as_secret_id(),
2080                    schema_id: obj.schema_id.unwrap(),
2081                    database_id: obj.database_id.unwrap(),
2082                    ..Default::default()
2083                })),
2084            }),
2085        }
2086    }
2087    NotificationInfo::ObjectGroup(PbObjectGroup {
2088        objects,
2089        dependencies: vec![],
2090    })
2091}
2092
2093pub fn extract_external_table_name_from_definition(table_definition: &str) -> Option<String> {
2094    let [mut definition]: [_; 1] = Parser::parse_sql(table_definition)
2095        .context("unable to parse table definition")
2096        .inspect_err(|e| {
2097            tracing::error!(
2098                target: "auto_schema_change",
2099                error = %e.as_report(),
2100                "failed to parse table definition")
2101        })
2102        .ok()?
2103        .try_into()
2104        .unwrap();
2105    if let SqlStatement::CreateTable { cdc_table_info, .. } = &mut definition {
2106        cdc_table_info
2107            .clone()
2108            .map(|cdc_table_info| cdc_table_info.external_table_name)
2109    } else {
2110        None
2111    }
2112}
2113
2114/// Columns listed in `column.exclude.list` in a CDC table definition. Empty if the definition
2115/// cannot be parsed or has no such option.
2116pub fn extract_cdc_excluded_columns_from_definition(table_definition: &str) -> HashSet<String> {
2117    let Ok(SqlStatement::CreateTable { with_options, .. }) =
2118        Parser::parse_exactly_one(table_definition)
2119    else {
2120        return HashSet::new();
2121    };
2122    with_options
2123        .iter()
2124        .find(|o| o.name.real_value() == CDC_COLUMN_EXCLUDE_LIST)
2125        .and_then(|o| match &o.value {
2126            SqlOptionValue::Value(Value::SingleQuotedString(s))
2127            | SqlOptionValue::Value(Value::CstyleEscapedString(CstyleEscapedString {
2128                value: s,
2129                ..
2130            })) => Some(parse_cdc_column_exclude_list(s)),
2131            _ => None,
2132        })
2133        .unwrap_or_default()
2134}
2135
2136/// `rename_relation` renames the target relation and its definition,
2137/// it commits the changes to the transaction and returns the updated relations and the old name.
2138pub async fn rename_relation(
2139    txn: &DatabaseTransaction,
2140    object_type: ObjectType,
2141    object_id: ObjectId,
2142    object_name: &str,
2143) -> MetaResult<(Vec<PbObject>, String)> {
2144    use sea_orm::ActiveModelTrait;
2145
2146    use crate::controller::rename::alter_relation_rename;
2147
2148    let mut to_update_relations = vec![];
2149    // rename relation.
2150    macro_rules! rename_relation {
2151        ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
2152            let (mut relation, obj) = $entity::find_by_id($object_id)
2153                .find_also_related(Object)
2154                .one(txn)
2155                .await?
2156                .unwrap();
2157            let obj = obj.unwrap();
2158            let old_name = relation.name.clone();
2159            relation.name = object_name.into();
2160            if obj.obj_type != ObjectType::View {
2161                relation.definition = alter_relation_rename(&relation.definition, object_name)
2162                    .with_context(|| format!("failed to rename {old_name}"))?;
2163            }
2164            let active_model = $table::ActiveModel {
2165                $identity: Set(relation.$identity),
2166                name: Set(object_name.into()),
2167                definition: Set(relation.definition.clone()),
2168                ..Default::default()
2169            };
2170            active_model.update(txn).await?;
2171            let streaming_job = streaming_job::Entity::find_by_id($object_id.as_raw_id())
2172                .one(txn)
2173                .await?;
2174            to_update_relations.push(PbObject {
2175                object_info: Some(PbObjectInfo::$entity(
2176                    ObjectModel(relation, obj, streaming_job).into(),
2177                )),
2178            });
2179            old_name
2180        }};
2181    }
2182    // TODO: check is there any thing to change for shared source?
2183    let old_name = match object_type {
2184        ObjectType::Table => {
2185            let associated_source_id: Option<SourceId> = Source::find()
2186                .select_only()
2187                .column(source::Column::SourceId)
2188                .filter(source::Column::OptionalAssociatedTableId.eq(object_id))
2189                .into_tuple()
2190                .one(txn)
2191                .await?;
2192            if let Some(source_id) = associated_source_id {
2193                rename_relation!(Source, source, source_id, source_id);
2194            }
2195            rename_relation!(Table, table, table_id, object_id.as_table_id())
2196        }
2197        ObjectType::Source => {
2198            rename_relation!(Source, source, source_id, object_id.as_source_id())
2199        }
2200        ObjectType::Sink => rename_relation!(Sink, sink, sink_id, object_id.as_sink_id()),
2201        ObjectType::Subscription => {
2202            rename_relation!(
2203                Subscription,
2204                subscription,
2205                subscription_id,
2206                object_id.as_subscription_id()
2207            )
2208        }
2209        ObjectType::View => rename_relation!(View, view, view_id, object_id.as_view_id()),
2210        ObjectType::Index => {
2211            let (mut index, obj) = Index::find_by_id(object_id.as_index_id())
2212                .find_also_related(Object)
2213                .one(txn)
2214                .await?
2215                .unwrap();
2216            let streaming_job = streaming_job::Entity::find_by_id(index.index_id.as_job_id())
2217                .one(txn)
2218                .await?;
2219            index.name = object_name.into();
2220            let index_table_id = index.index_table_id;
2221            let old_name = rename_relation!(Table, table, table_id, index_table_id);
2222
2223            // the name of index and its associated table is the same.
2224            let active_model = index::ActiveModel {
2225                index_id: sea_orm::ActiveValue::Set(index.index_id),
2226                name: sea_orm::ActiveValue::Set(object_name.into()),
2227                ..Default::default()
2228            };
2229            active_model.update(txn).await?;
2230            to_update_relations.push(PbObject {
2231                object_info: Some(PbObjectInfo::Index(
2232                    ObjectModel(index, obj.unwrap(), streaming_job).into(),
2233                )),
2234            });
2235            old_name
2236        }
2237        _ => unreachable!("only relation name can be altered."),
2238    };
2239
2240    Ok((to_update_relations, old_name))
2241}
2242
2243pub async fn get_database_resource_group<C>(txn: &C, database_id: DatabaseId) -> MetaResult<String>
2244where
2245    C: ConnectionTrait,
2246{
2247    let database_resource_group: Option<String> = Database::find_by_id(database_id)
2248        .select_only()
2249        .column(database::Column::ResourceGroup)
2250        .into_tuple()
2251        .one(txn)
2252        .await?
2253        .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))?;
2254
2255    Ok(database_resource_group.unwrap_or_else(|| DEFAULT_RESOURCE_GROUP.to_owned()))
2256}
2257
2258pub async fn get_existing_job_resource_group<C>(
2259    txn: &C,
2260    streaming_job_id: JobId,
2261) -> MetaResult<String>
2262where
2263    C: ConnectionTrait,
2264{
2265    let (job_specific_resource_group, database_resource_group): (Option<String>, Option<String>) =
2266        StreamingJob::find_by_id(streaming_job_id)
2267            .select_only()
2268            .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
2269            .join(JoinType::InnerJoin, object::Relation::Database2.def())
2270            .column(streaming_job::Column::SpecificResourceGroup)
2271            .column(database::Column::ResourceGroup)
2272            .into_tuple()
2273            .one(txn)
2274            .await?
2275            .ok_or_else(|| MetaError::catalog_id_not_found("streaming job", streaming_job_id))?;
2276
2277    Ok(job_specific_resource_group.unwrap_or_else(|| {
2278        database_resource_group.unwrap_or_else(|| DEFAULT_RESOURCE_GROUP.to_owned())
2279    }))
2280}
2281
2282pub fn filter_workers_by_resource_group(
2283    workers: &HashMap<WorkerId, WorkerNode>,
2284    resource_group: &str,
2285) -> BTreeSet<WorkerId> {
2286    workers
2287        .iter()
2288        .filter(|&(_, worker)| {
2289            worker
2290                .resource_group()
2291                .map(|node_label| node_label.as_str() == resource_group)
2292                .unwrap_or(false)
2293        })
2294        .map(|(id, _)| *id)
2295        .collect()
2296}
2297
2298/// `rename_relation_refer` updates the definition of relations that refer to the target one,
2299/// it commits the changes to the transaction and returns all the updated relations.
2300pub async fn rename_relation_refer(
2301    txn: &DatabaseTransaction,
2302    object_type: ObjectType,
2303    object_id: ObjectId,
2304    object_name: &str,
2305    old_name: &str,
2306) -> MetaResult<Vec<PbObject>> {
2307    use sea_orm::ActiveModelTrait;
2308
2309    use crate::controller::rename::alter_relation_rename_refs;
2310
2311    let mut to_update_relations = vec![];
2312    macro_rules! rename_relation_ref {
2313        ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
2314            let (mut relation, obj) = $entity::find_by_id($object_id)
2315                .find_also_related(Object)
2316                .one(txn)
2317                .await?
2318                .unwrap();
2319            relation.definition =
2320                alter_relation_rename_refs(&relation.definition, old_name, object_name)
2321                    .with_context(|| {
2322                        format!("failed to update the definition of {}", relation.name)
2323                    })?;
2324            let active_model = $table::ActiveModel {
2325                $identity: Set(relation.$identity),
2326                definition: Set(relation.definition.clone()),
2327                ..Default::default()
2328            };
2329            active_model.update(txn).await?;
2330            let streaming_job = streaming_job::Entity::find_by_id($object_id.as_raw_id())
2331                .one(txn)
2332                .await?;
2333            to_update_relations.push(PbObject {
2334                object_info: Some(PbObjectInfo::$entity(
2335                    ObjectModel(relation, obj.unwrap(), streaming_job).into(),
2336                )),
2337            });
2338        }};
2339    }
2340    let objs = get_referring_objects(object_id, object_type, txn).await?;
2341
2342    for obj in objs {
2343        match obj.obj_type {
2344            ObjectType::Table => {
2345                rename_relation_ref!(Table, table, table_id, obj.oid.as_table_id())
2346            }
2347            ObjectType::Sink => {
2348                rename_relation_ref!(Sink, sink, sink_id, obj.oid.as_sink_id())
2349            }
2350            // A CDC table source refers to the shared CDC source it reads from.
2351            ObjectType::Source => {
2352                rename_relation_ref!(Source, source, source_id, obj.oid.as_source_id())
2353            }
2354            ObjectType::Subscription => {
2355                rename_relation_ref!(
2356                    Subscription,
2357                    subscription,
2358                    subscription_id,
2359                    obj.oid.as_subscription_id()
2360                )
2361            }
2362            ObjectType::View => rename_relation_ref!(View, view, view_id, obj.oid.as_view_id()),
2363            ObjectType::Index => {
2364                let index_table_id: Option<TableId> = Index::find_by_id(obj.oid.as_index_id())
2365                    .select_only()
2366                    .column(index::Column::IndexTableId)
2367                    .into_tuple()
2368                    .one(txn)
2369                    .await?;
2370                rename_relation_ref!(Table, table, table_id, index_table_id.unwrap());
2371            }
2372            _ => {
2373                bail!(
2374                    "only the table, source, sink, subscription, view and index will depend on other objects."
2375                )
2376            }
2377        }
2378    }
2379
2380    Ok(to_update_relations)
2381}
2382
2383/// Validate that subscription can be safely deleted, meeting any of the following conditions:
2384/// 1. The upstream table is not referred to by any cross-db mv.
2385/// 2. After deleting the subscription, the upstream table still has at least one subscription.
2386pub async fn validate_subscription_deletion<C>(
2387    txn: &C,
2388    subscription_id: SubscriptionId,
2389) -> MetaResult<()>
2390where
2391    C: ConnectionTrait,
2392{
2393    let upstream_table_id: ObjectId = Subscription::find_by_id(subscription_id)
2394        .select_only()
2395        .column(subscription::Column::DependentTableId)
2396        .into_tuple()
2397        .one(txn)
2398        .await?
2399        .ok_or_else(|| MetaError::catalog_id_not_found("subscription", subscription_id))?;
2400
2401    let cnt = Subscription::find()
2402        .filter(subscription::Column::DependentTableId.eq(upstream_table_id))
2403        .count(txn)
2404        .await?;
2405    if cnt > 1 {
2406        // Ensure that at least one subscription is remained for the upstream table
2407        // once the subscription is dropped.
2408        return Ok(());
2409    }
2410
2411    // Ensure that the upstream table is not referred by any cross-db mv.
2412    let obj_alias = Alias::new("o1");
2413    let used_by_alias = Alias::new("o2");
2414    let count = ObjectDependency::find()
2415        .join_as(
2416            JoinType::InnerJoin,
2417            object_dependency::Relation::Object2.def(),
2418            obj_alias.clone(),
2419        )
2420        .join_as(
2421            JoinType::InnerJoin,
2422            object_dependency::Relation::Object1.def(),
2423            used_by_alias.clone(),
2424        )
2425        .filter(
2426            object_dependency::Column::Oid
2427                .eq(upstream_table_id)
2428                .and(object_dependency::Column::UsedBy.ne(subscription_id))
2429                .and(
2430                    Expr::col((obj_alias, object::Column::DatabaseId))
2431                        .ne(Expr::col((used_by_alias, object::Column::DatabaseId))),
2432                ),
2433        )
2434        .count(txn)
2435        .await?;
2436
2437    if count != 0 {
2438        return Err(MetaError::permission_denied(format!(
2439            "Referenced by {} cross-db objects.",
2440            count
2441        )));
2442    }
2443
2444    Ok(())
2445}
2446
2447pub async fn fetch_target_fragments<C>(
2448    txn: &C,
2449    src_fragment_id: impl IntoIterator<Item = FragmentId>,
2450) -> MetaResult<HashMap<FragmentId, Vec<FragmentId>>>
2451where
2452    C: ConnectionTrait,
2453{
2454    let source_target_fragments: Vec<(FragmentId, FragmentId)> = FragmentRelation::find()
2455        .select_only()
2456        .columns([
2457            fragment_relation::Column::SourceFragmentId,
2458            fragment_relation::Column::TargetFragmentId,
2459        ])
2460        .filter(fragment_relation::Column::SourceFragmentId.is_in(src_fragment_id))
2461        .into_tuple()
2462        .all(txn)
2463        .await?;
2464
2465    let source_target_fragments = source_target_fragments.into_iter().into_group_map();
2466
2467    Ok(source_target_fragments)
2468}
2469
2470pub async fn get_sink_fragment_by_ids<C>(
2471    txn: &C,
2472    sink_ids: Vec<SinkId>,
2473) -> MetaResult<HashMap<SinkId, FragmentId>>
2474where
2475    C: ConnectionTrait,
2476{
2477    let sink_num = sink_ids.len();
2478    let sink_fragment_ids: Vec<(SinkId, FragmentId)> = Fragment::find()
2479        .select_only()
2480        .columns([fragment::Column::JobId, fragment::Column::FragmentId])
2481        .filter(
2482            fragment::Column::JobId
2483                .is_in(sink_ids)
2484                .and(FragmentTypeMask::intersects(FragmentTypeFlag::Sink)),
2485        )
2486        .into_tuple()
2487        .all(txn)
2488        .await?;
2489
2490    if sink_fragment_ids.len() != sink_num {
2491        return Err(anyhow::anyhow!(
2492            "expected exactly one sink fragment for each sink, but got {} fragments for {} sinks",
2493            sink_fragment_ids.len(),
2494            sink_num
2495        )
2496        .into());
2497    }
2498
2499    Ok(sink_fragment_ids.into_iter().collect())
2500}
2501
2502pub async fn has_table_been_migrated<C>(txn: &C, table_id: TableId) -> MetaResult<bool>
2503where
2504    C: ConnectionTrait,
2505{
2506    let mview_fragment: Vec<i32> = Fragment::find()
2507        .select_only()
2508        .column(fragment::Column::FragmentTypeMask)
2509        .filter(
2510            fragment::Column::JobId
2511                .eq(table_id)
2512                .and(FragmentTypeMask::intersects(FragmentTypeFlag::Mview)),
2513        )
2514        .into_tuple()
2515        .all(txn)
2516        .await?;
2517
2518    let mview_fragment_len = mview_fragment.len();
2519    if mview_fragment_len != 1 {
2520        bail!(
2521            "expected exactly one mview fragment for table {}, found {}",
2522            table_id,
2523            mview_fragment_len
2524        );
2525    }
2526
2527    let mview_fragment = mview_fragment.into_iter().next().unwrap();
2528    let migrated =
2529        FragmentTypeMask::from(mview_fragment).contains(FragmentTypeFlag::UpstreamSinkUnion);
2530
2531    Ok(migrated)
2532}
2533
2534pub async fn check_if_belongs_to_iceberg_table<C>(txn: &C, job_id: JobId) -> MetaResult<bool>
2535where
2536    C: ConnectionTrait,
2537{
2538    if let Some(engine) = Table::find_by_id(job_id.as_mv_table_id())
2539        .select_only()
2540        .column(table::Column::Engine)
2541        .into_tuple::<table::Engine>()
2542        .one(txn)
2543        .await?
2544        && engine == table::Engine::Iceberg
2545    {
2546        return Ok(true);
2547    }
2548    if let Some(parent_oid) = Object::find_by_id(job_id)
2549        .select_only()
2550        .column(object::Column::BelongToOid)
2551        .into_tuple::<Option<ObjectId>>()
2552        .one(txn)
2553        .await?
2554        .flatten()
2555        && Table::find_by_id(parent_oid.as_table_id())
2556            .filter(table::Column::Engine.eq(table::Engine::Iceberg))
2557            .one(txn)
2558            .await?
2559            .is_some()
2560    {
2561        return Ok(true);
2562    }
2563    Ok(false)
2564}
2565
2566pub async fn find_dirty_iceberg_table_jobs<C>(
2567    txn: &C,
2568    database_id: Option<DatabaseId>,
2569) -> MetaResult<Vec<PartialObject>>
2570where
2571    C: ConnectionTrait,
2572{
2573    let mut filter_condition = streaming_job::Column::JobStatus
2574        .ne(JobStatus::Created)
2575        .and(object::Column::ObjType.is_in([ObjectType::Table, ObjectType::Sink]))
2576        .and(streaming_job::Column::CreateType.eq(CreateType::Background));
2577    if let Some(database_id) = database_id {
2578        filter_condition = filter_condition.and(object::Column::DatabaseId.eq(database_id));
2579    }
2580    let creating_table_sink_jobs: Vec<PartialObject> = StreamingJob::find()
2581        .select_only()
2582        .columns([
2583            object::Column::Oid,
2584            object::Column::ObjType,
2585            object::Column::SchemaId,
2586            object::Column::DatabaseId,
2587        ])
2588        .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
2589        .filter(filter_condition)
2590        .into_partial_model()
2591        .all(txn)
2592        .await?;
2593
2594    let mut dirty_iceberg_table_jobs = vec![];
2595    for job in creating_table_sink_jobs {
2596        if check_if_belongs_to_iceberg_table(txn, job.oid.as_job_id()).await? {
2597            tracing::info!("Found dirty iceberg job with id: {}", job.oid);
2598            dirty_iceberg_table_jobs.push(job);
2599        }
2600    }
2601
2602    Ok(dirty_iceberg_table_jobs)
2603}
2604
2605pub fn build_select_node_list(
2606    from: &[ColumnCatalog],
2607    to: &[ColumnCatalog],
2608) -> MetaResult<Vec<PbExprNode>> {
2609    let mut exprs = Vec::with_capacity(to.len());
2610    let idx_by_col_id = from
2611        .iter()
2612        .enumerate()
2613        .map(|(idx, col)| (col.column_desc.as_ref().unwrap().column_id, idx))
2614        .collect::<HashMap<_, _>>();
2615
2616    for to_col in to {
2617        let to_col = to_col.column_desc.as_ref().unwrap();
2618        let to_col_type_ref = to_col.column_type.as_ref().unwrap();
2619        let to_col_type = DataType::from(to_col_type_ref);
2620        if let Some(from_idx) = idx_by_col_id.get(&to_col.column_id) {
2621            let from_col_type = DataType::from(
2622                from[*from_idx]
2623                    .column_desc
2624                    .as_ref()
2625                    .unwrap()
2626                    .column_type
2627                    .as_ref()
2628                    .unwrap(),
2629            );
2630            if !to_col_type.equals_datatype(&from_col_type) {
2631                return Err(anyhow!(
2632                    "Column type mismatch: {:?} != {:?}",
2633                    from_col_type,
2634                    to_col_type
2635                )
2636                .into());
2637            }
2638            exprs.push(PbExprNode {
2639                function_type: expr_node::Type::Unspecified.into(),
2640                return_type: Some(to_col_type_ref.clone()),
2641                rex_node: Some(expr_node::RexNode::InputRef(*from_idx as _)),
2642            });
2643        } else {
2644            let to_default_node =
2645                if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
2646                    expr,
2647                    ..
2648                })) = &to_col.generated_or_default_column
2649                {
2650                    expr.clone().unwrap()
2651                } else {
2652                    let null = Datum::None.to_protobuf();
2653                    PbExprNode {
2654                        function_type: expr_node::Type::Unspecified.into(),
2655                        return_type: Some(to_col_type_ref.clone()),
2656                        rex_node: Some(expr_node::RexNode::Constant(null)),
2657                    }
2658                };
2659            exprs.push(to_default_node);
2660        }
2661    }
2662
2663    Ok(exprs)
2664}
2665
2666#[derive(Clone, Debug, Default)]
2667pub struct StreamingJobExtraInfo {
2668    pub timezone: Option<String>,
2669    pub config_override: Arc<str>,
2670    pub job_definition: String,
2671    pub backfill_orders: Option<BackfillOrders>,
2672    pub refresh_interval_sec: Option<u64>,
2673}
2674
2675impl StreamingJobExtraInfo {
2676    pub fn stream_context(&self) -> StreamContext {
2677        StreamContext {
2678            timezone: self.timezone.clone(),
2679            config_override: self.config_override.clone(),
2680        }
2681    }
2682}
2683
2684/// Tuple of (`job_id`, `timezone`, `config_override`, `backfill_orders`, `refresh_interval_sec`)
2685type StreamingJobExtraInfoRow = (
2686    JobId,
2687    Option<String>,
2688    Option<String>,
2689    Option<BackfillOrders>,
2690    Option<i64>,
2691);
2692
2693pub async fn get_streaming_job_extra_info<C>(
2694    txn: &C,
2695    job_ids: Vec<JobId>,
2696) -> MetaResult<HashMap<JobId, StreamingJobExtraInfo>>
2697where
2698    C: ConnectionTrait,
2699{
2700    let pairs: Vec<StreamingJobExtraInfoRow> = StreamingJob::find()
2701        .select_only()
2702        .columns([
2703            streaming_job::Column::JobId,
2704            streaming_job::Column::Timezone,
2705            streaming_job::Column::ConfigOverride,
2706            streaming_job::Column::BackfillOrders,
2707            streaming_job::Column::RefreshIntervalSec,
2708        ])
2709        .filter(streaming_job::Column::JobId.is_in(job_ids.clone()))
2710        .into_tuple()
2711        .all(txn)
2712        .await?;
2713
2714    let job_ids = job_ids.into_iter().collect();
2715
2716    let mut definitions = resolve_streaming_job_definition(txn, &job_ids).await?;
2717
2718    let result = pairs
2719        .into_iter()
2720        .map(
2721            |(job_id, timezone, config_override, backfill_orders, refresh_interval_sec)| {
2722                let job_definition = definitions.remove(&job_id).unwrap_or_default();
2723                (
2724                    job_id,
2725                    StreamingJobExtraInfo {
2726                        timezone,
2727                        config_override: config_override.unwrap_or_default().into(),
2728                        job_definition,
2729                        backfill_orders,
2730                        refresh_interval_sec: refresh_interval_sec.map(|s| s as u64),
2731                    },
2732                )
2733            },
2734        )
2735        .collect();
2736
2737    Ok(result)
2738}
2739
2740#[cfg(test)]
2741mod tests {
2742    use super::*;
2743
2744    #[test]
2745    fn test_extract_cdc_table_name() {
2746        let ddl1 = "CREATE TABLE t1 () FROM pg_source TABLE 'public.t1'";
2747        let ddl2 = "CREATE TABLE t2 (v1 int) FROM pg_source TABLE 'mydb.t2'";
2748        assert_eq!(
2749            extract_external_table_name_from_definition(ddl1),
2750            Some("public.t1".into())
2751        );
2752        assert_eq!(
2753            extract_external_table_name_from_definition(ddl2),
2754            Some("mydb.t2".into())
2755        );
2756    }
2757}