1use std::collections::{BTreeSet, HashMap, HashSet};
16use std::sync::Arc;
17
18use anyhow::{Context, anyhow};
19use itertools::Itertools;
20use risingwave_common::bitmap::Bitmap;
21use risingwave_common::catalog::{
22 FragmentTypeFlag, FragmentTypeMask, ICEBERG_SINK_PREFIX, ICEBERG_SOURCE_PREFIX,
23};
24use risingwave_common::hash::{ActorMapping, VnodeBitmapExt, WorkerSlotId, WorkerSlotMapping};
25use risingwave_common::id::{JobId, SubscriptionId};
26use risingwave_common::types::{DataType, Datum};
27use risingwave_common::util::value_encoding::DatumToProtoExt;
28use risingwave_common::util::worker_util::DEFAULT_RESOURCE_GROUP;
29use risingwave_common::{bail, hash};
30use risingwave_connector::WithOptionsSecResolved;
31use risingwave_connector::source::cdc::{CDC_COLUMN_EXCLUDE_LIST, parse_cdc_column_exclude_list};
32use risingwave_meta_model::fragment::DistributionType;
33use risingwave_meta_model::object::ObjectType;
34use risingwave_meta_model::prelude::*;
35use risingwave_meta_model::streaming_job::BackfillOrders;
36use risingwave_meta_model::table::TableType;
37use risingwave_meta_model::user_privilege::Action;
38use risingwave_meta_model::{
39 ActorId, ColumnCatalogArray, CreateType, DataTypeArray, DatabaseId, DispatcherType, FragmentId,
40 JobStatus, ObjectId, PrivilegeId, SchemaId, SinkId, SourceId, StreamNode, StreamSourceInfo,
41 TableId, TableIdArray, UserId, WorkerId, connection, database, fragment, fragment_relation,
42 function, index, object, object_dependency, schema, secret, sink, source, streaming_job,
43 subscription, table, user, user_default_privilege, user_privilege, view,
44};
45use risingwave_meta_model_migration::WithQuery;
46use risingwave_pb::catalog::{
47 PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource,
48 PbSubscription, PbTable, PbView,
49};
50use risingwave_pb::common::{PbObjectType, WorkerNode};
51use risingwave_pb::expr::{PbExprNode, expr_node};
52use risingwave_pb::meta::object::PbObjectInfo;
53use risingwave_pb::meta::subscribe_response::Info as NotificationInfo;
54use risingwave_pb::meta::{
55 ObjectDependency as PbObjectDependency, PbFragmentWorkerSlotMapping, PbObject, PbObjectGroup,
56};
57use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
58use risingwave_pb::plan_common::{ColumnCatalog, DefaultColumnDesc};
59use risingwave_pb::secret::PbSecretRef;
60use risingwave_pb::secret::secret_ref::PbRefAsType;
61use risingwave_pb::stream_plan::{PbDispatchOutputMapping, PbDispatcher, PbDispatcherType};
62use risingwave_pb::user::grant_privilege::{PbActionWithGrantOption, PbObject as PbGrantObject};
63use risingwave_pb::user::{PbAction, PbGrantPrivilege, PbUserInfo};
64use risingwave_sqlparser::ast::{
65 CstyleEscapedString, Ident, ObjectName, SecretRefAsType, SecretRefValue, SqlOption,
66 SqlOptionValue, Statement as SqlStatement, Value,
67};
68use risingwave_sqlparser::parser::Parser;
69use sea_orm::sea_query::{
70 Alias, CommonTableExpression, Expr, OnConflict, Query, QueryStatementBuilder, SelectStatement,
71 UnionType, WithClause,
72};
73use sea_orm::{
74 ColumnTrait, ConnectionTrait, DatabaseTransaction, DerivePartialModel, EntityTrait,
75 FromQueryResult, IntoActiveModel, JoinType, Order, PaginatorTrait, QueryFilter, QuerySelect,
76 RelationTrait, Set, Statement,
77};
78use thiserror_ext::AsReport;
79use tracing::warn;
80
81use crate::barrier::SharedFragmentInfo;
82use crate::controller::ObjectModel;
83use crate::controller::fragment::FragmentTypeMaskExt;
84use crate::controller::scale::resolve_streaming_job_definition;
85use crate::model::{FragmentDownstreamRelation, StreamContext};
86use crate::{MetaError, MetaResult};
87
88pub fn construct_obj_dependency_query(obj_id: ObjectId) -> WithQuery {
113 let cte_alias = Alias::new("used_by_object_ids");
114 let cte_return_alias = Alias::new("used_by");
115
116 let mut base_query = SelectStatement::new()
117 .column(object_dependency::Column::UsedBy)
118 .from(ObjectDependency)
119 .and_where(object_dependency::Column::Oid.eq(obj_id))
120 .to_owned();
121
122 let belonged_obj_query = SelectStatement::new()
123 .column(object::Column::Oid)
124 .from(Object)
125 .and_where(
126 object::Column::DatabaseId
127 .eq(obj_id)
128 .or(object::Column::SchemaId.eq(obj_id)),
129 )
130 .to_owned();
131
132 let cte_referencing = Query::select()
133 .column((ObjectDependency, object_dependency::Column::UsedBy))
134 .from(ObjectDependency)
135 .inner_join(
136 cte_alias.clone(),
137 Expr::col((cte_alias.clone(), cte_return_alias.clone()))
138 .equals(object_dependency::Column::Oid),
139 )
140 .to_owned();
141
142 let mut common_table_expr = CommonTableExpression::new();
143 common_table_expr
144 .query(
145 base_query
146 .union(UnionType::All, belonged_obj_query)
147 .union(UnionType::All, cte_referencing)
148 .to_owned(),
149 )
150 .column(cte_return_alias.clone())
151 .table_name(cte_alias.clone());
152
153 SelectStatement::new()
154 .distinct()
155 .columns([
156 object::Column::Oid,
157 object::Column::ObjType,
158 object::Column::SchemaId,
159 object::Column::DatabaseId,
160 ])
161 .from(cte_alias.clone())
162 .inner_join(
163 Object,
164 Expr::col((cte_alias, cte_return_alias)).equals(object::Column::Oid),
165 )
166 .order_by(object::Column::Oid, Order::Desc)
167 .to_owned()
168 .with(
169 WithClause::new()
170 .recursive(true)
171 .cte(common_table_expr)
172 .to_owned(),
173 )
174}
175
176fn to_pb_object_type(obj_type: ObjectType) -> PbObjectType {
177 match obj_type {
178 ObjectType::Database => PbObjectType::Database,
179 ObjectType::Schema => PbObjectType::Schema,
180 ObjectType::Table => PbObjectType::Table,
181 ObjectType::Source => PbObjectType::Source,
182 ObjectType::Sink => PbObjectType::Sink,
183 ObjectType::View => PbObjectType::View,
184 ObjectType::Index => PbObjectType::Index,
185 ObjectType::Function => PbObjectType::Function,
186 ObjectType::Connection => PbObjectType::Connection,
187 ObjectType::Subscription => PbObjectType::Subscription,
188 ObjectType::Secret => PbObjectType::Secret,
189 }
190}
191
192async fn list_object_dependencies_impl(
194 txn: &DatabaseTransaction,
195 object_id: Option<ObjectId>,
196 include_creating: bool,
197) -> MetaResult<Vec<PbObjectDependency>> {
198 let referenced_alias = Alias::new("referenced_obj");
199 let mut query = ObjectDependency::find()
200 .select_only()
201 .columns([
202 object_dependency::Column::Oid,
203 object_dependency::Column::UsedBy,
204 ])
205 .column_as(
206 Expr::col((referenced_alias.clone(), object::Column::ObjType)),
207 "referenced_obj_type",
208 )
209 .join(
210 JoinType::InnerJoin,
211 object_dependency::Relation::Object1.def(),
212 )
213 .join_as(
214 JoinType::InnerJoin,
215 object_dependency::Relation::Object2.def(),
216 referenced_alias.clone(),
217 );
218 if let Some(object_id) = object_id {
219 query = query.filter(object_dependency::Column::UsedBy.eq(object_id));
220 }
221 let mut obj_dependencies: Vec<PbObjectDependency> = query
222 .into_tuple()
223 .all(txn)
224 .await?
225 .into_iter()
226 .map(|(oid, used_by, referenced_type)| PbObjectDependency {
227 object_id: used_by,
228 referenced_object_id: oid,
229 referenced_object_type: to_pb_object_type(referenced_type) as i32,
230 })
231 .collect();
232
233 let mut sink_query = Sink::find()
234 .select_only()
235 .columns([sink::Column::SinkId, sink::Column::TargetTable])
236 .filter(sink::Column::TargetTable.is_not_null());
237 if let Some(object_id) = object_id {
238 sink_query = sink_query.filter(
239 sink::Column::SinkId
240 .eq(object_id)
241 .or(sink::Column::TargetTable.eq(object_id)),
242 );
243 }
244 let sink_dependencies: Vec<(SinkId, TableId)> = sink_query.into_tuple().all(txn).await?;
245 obj_dependencies.extend(sink_dependencies.into_iter().map(|(sink_id, table_id)| {
246 PbObjectDependency {
247 object_id: table_id.into(),
248 referenced_object_id: sink_id.into(),
249 referenced_object_type: PbObjectType::Sink as i32,
250 }
251 }));
252
253 if !include_creating {
254 let mut streaming_job_ids = obj_dependencies
255 .iter()
256 .map(|dependency| dependency.object_id)
257 .collect_vec();
258 streaming_job_ids.sort_unstable();
259 streaming_job_ids.dedup();
260
261 if !streaming_job_ids.is_empty() {
262 let non_created_jobs: HashSet<JobId> = StreamingJob::find()
263 .select_only()
264 .columns([streaming_job::Column::JobId])
265 .filter(
266 streaming_job::Column::JobId
267 .is_in(streaming_job_ids)
268 .and(streaming_job::Column::JobStatus.ne(JobStatus::Created)),
269 )
270 .into_tuple()
271 .all(txn)
272 .await?
273 .into_iter()
274 .collect();
275
276 if !non_created_jobs.is_empty() {
277 obj_dependencies.retain(|dependency| {
278 !non_created_jobs.contains(&dependency.object_id.as_job_id())
279 });
280 }
281 }
282 }
283
284 Ok(obj_dependencies)
285}
286
287pub async fn list_object_dependencies(
289 txn: &DatabaseTransaction,
290 include_creating: bool,
291) -> MetaResult<Vec<PbObjectDependency>> {
292 list_object_dependencies_impl(txn, None, include_creating).await
293}
294
295pub async fn list_object_dependencies_by_object_id(
297 txn: &DatabaseTransaction,
298 object_id: ObjectId,
299) -> MetaResult<Vec<PbObjectDependency>> {
300 list_object_dependencies_impl(txn, Some(object_id), true).await
301}
302
303pub fn construct_sink_cycle_check_query(
328 target_table: ObjectId,
329 dependent_objects: Vec<ObjectId>,
330) -> WithQuery {
331 let cte_alias = Alias::new("used_by_object_ids_with_sink");
332 let depend_alias = Alias::new("obj_dependency_with_sink");
333
334 let mut base_query = SelectStatement::new()
335 .columns([
336 object_dependency::Column::Oid,
337 object_dependency::Column::UsedBy,
338 ])
339 .from(ObjectDependency)
340 .and_where(object_dependency::Column::Oid.eq(target_table))
341 .to_owned();
342
343 let query_sink_deps = SelectStatement::new()
344 .columns([sink::Column::SinkId, sink::Column::TargetTable])
345 .from(Sink)
346 .and_where(sink::Column::TargetTable.is_not_null())
347 .to_owned();
348
349 let cte_referencing = Query::select()
350 .column((depend_alias.clone(), object_dependency::Column::Oid))
351 .column((depend_alias.clone(), object_dependency::Column::UsedBy))
352 .from_subquery(
353 SelectStatement::new()
354 .columns([
355 object_dependency::Column::Oid,
356 object_dependency::Column::UsedBy,
357 ])
358 .from(ObjectDependency)
359 .union(UnionType::All, query_sink_deps)
360 .to_owned(),
361 depend_alias.clone(),
362 )
363 .inner_join(
364 cte_alias.clone(),
365 Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy))
366 .eq(Expr::col((depend_alias, object_dependency::Column::Oid))),
367 )
368 .and_where(
369 Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy)).ne(Expr::col((
370 cte_alias.clone(),
371 object_dependency::Column::Oid,
372 ))),
373 )
374 .to_owned();
375
376 let mut common_table_expr = CommonTableExpression::new();
377 common_table_expr
378 .query(base_query.union(UnionType::All, cte_referencing).to_owned())
379 .columns([
380 object_dependency::Column::Oid,
381 object_dependency::Column::UsedBy,
382 ])
383 .table_name(cte_alias.clone());
384
385 SelectStatement::new()
386 .expr(Expr::col((cte_alias.clone(), object_dependency::Column::UsedBy)).count())
387 .from(cte_alias.clone())
388 .and_where(
389 Expr::col((cte_alias, object_dependency::Column::UsedBy)).is_in(dependent_objects),
390 )
391 .to_owned()
392 .with(
393 WithClause::new()
394 .recursive(true)
395 .cte(common_table_expr)
396 .to_owned(),
397 )
398}
399
400#[derive(Clone, DerivePartialModel, FromQueryResult, Debug)]
401#[sea_orm(entity = "Object")]
402pub struct PartialObject {
403 pub oid: ObjectId,
404 pub obj_type: ObjectType,
405 pub schema_id: Option<SchemaId>,
406 pub database_id: Option<DatabaseId>,
407}
408
409impl From<object::Model> for PartialObject {
410 fn from(object: object::Model) -> Self {
411 Self {
412 oid: object.oid,
413 obj_type: object.obj_type,
414 schema_id: object.schema_id,
415 database_id: object.database_id,
416 }
417 }
418}
419
420pub async fn get_belong_objects<C>(db: &C, object_id: ObjectId) -> MetaResult<Vec<object::Model>>
421where
422 C: ConnectionTrait,
423{
424 get_belong_objects_by_ids(db, [object_id]).await
425}
426
427pub async fn get_belong_objects_by_ids<C>(
429 db: &C,
430 object_ids: impl IntoIterator<Item = ObjectId>,
431) -> MetaResult<Vec<object::Model>>
432where
433 C: ConnectionTrait,
434{
435 let mut parents = object_ids.into_iter().collect_vec();
436 let mut visited = parents.iter().copied().collect::<HashSet<_>>();
437 let mut objects = vec![];
438 loop {
439 let children = Object::find()
440 .filter(object::Column::BelongToOid.is_in(parents))
441 .all(db)
442 .await?
443 .into_iter()
444 .filter(|object| visited.insert(object.oid))
445 .collect_vec();
446 if children.is_empty() {
447 break;
448 }
449
450 parents = children.iter().map(|object| object.oid).collect();
451 objects.extend(children);
452 }
453
454 Ok(objects)
455}
456
457#[derive(Clone, DerivePartialModel, FromQueryResult)]
458#[sea_orm(entity = "Fragment")]
459pub struct PartialFragmentStateTables {
460 pub fragment_id: FragmentId,
461 pub job_id: ObjectId,
462 pub state_table_ids: TableIdArray,
463}
464
465#[derive(Clone, Eq, PartialEq, Debug)]
466pub struct PartialActorLocation {
467 pub actor_id: ActorId,
468 pub fragment_id: FragmentId,
469 pub worker_id: WorkerId,
470}
471
472#[derive(FromQueryResult, Debug, Eq, PartialEq, Clone)]
473pub struct FragmentDesc {
474 pub fragment_id: FragmentId,
475 pub job_id: JobId,
476 pub fragment_type_mask: i32,
477 pub distribution_type: DistributionType,
478 pub state_table_ids: TableIdArray,
479 pub parallelism: i64,
480 pub vnode_count: i32,
481 pub stream_node: StreamNode,
482 pub parallelism_policy: String,
483}
484
485pub async fn get_referring_objects_cascade<C>(
487 obj_id: ObjectId,
488 object_type: ObjectType,
489 db: &C,
490) -> MetaResult<Vec<PartialObject>>
491where
492 C: ConnectionTrait,
493{
494 let query = construct_obj_dependency_query(obj_id);
495 let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
496 let mut objects = PartialObject::find_by_statement(Statement::from_sql_and_values(
497 db.get_database_backend(),
498 sql,
499 values,
500 ))
501 .all(db)
502 .await?;
503
504 let target_objects = std::iter::once((obj_id, object_type))
505 .chain(objects.iter().map(|object| (object.oid, object.obj_type)))
506 .collect_vec();
507 let mut existing_object_ids: HashSet<ObjectId> =
508 objects.iter().map(|object| object.oid).collect();
509 for (target_id, target_type) in target_objects {
510 let incoming_sink_objects =
511 get_incoming_sink_objects_for_target(target_id, target_type, db).await?;
512 for incoming_sink_object in incoming_sink_objects {
513 if existing_object_ids.insert(incoming_sink_object.oid) {
514 objects.push(incoming_sink_object);
515 }
516 }
517 }
518 Ok(objects)
519}
520
521pub async fn check_sink_into_table_cycle<C>(
523 target_table: ObjectId,
524 dependent_objs: Vec<ObjectId>,
525 db: &C,
526) -> MetaResult<bool>
527where
528 C: ConnectionTrait,
529{
530 if dependent_objs.is_empty() {
531 return Ok(false);
532 }
533
534 if dependent_objs.contains(&target_table) {
536 return Ok(true);
537 }
538
539 let query = construct_sink_cycle_check_query(target_table, dependent_objs);
540 let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
541
542 let res = db
543 .query_one(Statement::from_sql_and_values(
544 db.get_database_backend(),
545 sql,
546 values,
547 ))
548 .await?
549 .unwrap();
550
551 let cnt: i64 = res.try_get_by(0)?;
552
553 Ok(cnt != 0)
554}
555
556pub async fn format_with_option_secret_resolved(
563 txn: &DatabaseTransaction,
564 options_with_secret: &WithOptionsSecResolved,
565) -> MetaResult<Vec<SqlOption>> {
566 let mut options = Vec::new();
567 for (k, v) in options_with_secret.as_plaintext() {
568 let sql_option = SqlOption::try_from((k, v))
569 .map_err(|e| MetaError::invalid_parameter(e.to_report_string()))?;
570 options.push(sql_option);
571 }
572 for (name, secret_ref) in options_with_secret.as_secret() {
573 let secret_ref_value = resolve_secret_ref_value(txn, secret_ref).await?;
574 options.push(SqlOption::from_secret_ref(name, secret_ref_value));
575 }
576 Ok(options)
577}
578
579async fn resolve_secret_ref_value(
580 txn: &DatabaseTransaction,
581 secret_ref: &PbSecretRef,
582) -> MetaResult<SecretRefValue> {
583 let (secret, object) = Secret::find_by_id(secret_ref.secret_id)
584 .find_also_related(Object)
585 .one(txn)
586 .await?
587 .ok_or_else(|| MetaError::catalog_id_not_found("secret", secret_ref.secret_id))?;
588 let object =
589 object.ok_or_else(|| MetaError::catalog_id_not_found("object", secret_ref.secret_id))?;
590 let schema_id = object.schema_id.ok_or_else(|| {
591 MetaError::invalid_parameter(format!(
592 "secret {} does not belong to a schema",
593 secret_ref.secret_id
594 ))
595 })?;
596 let schema = Schema::find_by_id(schema_id)
597 .one(txn)
598 .await?
599 .ok_or_else(|| MetaError::catalog_id_not_found("schema", schema_id))?;
600
601 let ref_as = match PbRefAsType::try_from(secret_ref.ref_as) {
602 Ok(PbRefAsType::File) => SecretRefAsType::File,
603 Ok(PbRefAsType::Text | PbRefAsType::Unspecified) => SecretRefAsType::Text,
604 Err(_) => {
605 return Err(MetaError::invalid_parameter(format!(
606 "invalid reference type {} for secret {}",
607 secret_ref.ref_as, secret_ref.secret_id
608 )));
609 }
610 };
611
612 Ok(SecretRefValue {
613 secret_name: ObjectName(vec![
614 Ident::from_real_value(&schema.name),
615 Ident::from_real_value(&secret.name),
616 ]),
617 ref_as,
618 })
619}
620
621pub async fn ensure_object_id<C>(
623 object_type: ObjectType,
624 obj_id: impl Into<ObjectId>,
625 db: &C,
626) -> MetaResult<()>
627where
628 C: ConnectionTrait,
629{
630 let obj_id = obj_id.into();
631 let count = Object::find_by_id(obj_id).count(db).await?;
632 if count == 0 {
633 return Err(MetaError::catalog_id_not_found(
634 object_type.as_str(),
635 obj_id,
636 ));
637 }
638 Ok(())
639}
640
641pub async fn ensure_job_not_canceled<C>(job_id: JobId, db: &C) -> MetaResult<()>
642where
643 C: ConnectionTrait,
644{
645 let count = Object::find_by_id(job_id).count(db).await?;
646 if count == 0 {
647 return Err(MetaError::cancelled(format!(
648 "job {} might be cancelled manually or by recovery",
649 job_id
650 )));
651 }
652 Ok(())
653}
654
655pub async fn ensure_user_id<C>(user_id: UserId, db: &C) -> MetaResult<()>
657where
658 C: ConnectionTrait,
659{
660 let count = User::find_by_id(user_id).count(db).await?;
661 if count == 0 {
662 return Err(anyhow!("user {} was concurrently dropped", user_id).into());
663 }
664 Ok(())
665}
666
667pub async fn check_database_name_duplicate<C>(name: &str, db: &C) -> MetaResult<()>
669where
670 C: ConnectionTrait,
671{
672 let count = Database::find()
673 .filter(database::Column::Name.eq(name))
674 .count(db)
675 .await?;
676 if count > 0 {
677 assert_eq!(count, 1);
678 return Err(MetaError::catalog_duplicated("database", name));
679 }
680 Ok(())
681}
682
683pub async fn check_function_signature_duplicate<C>(
685 pb_function: &PbFunction,
686 db: &C,
687) -> MetaResult<()>
688where
689 C: ConnectionTrait,
690{
691 let count = Function::find()
692 .inner_join(Object)
693 .filter(
694 object::Column::DatabaseId
695 .eq(pb_function.database_id)
696 .and(object::Column::SchemaId.eq(pb_function.schema_id))
697 .and(function::Column::Name.eq(&pb_function.name))
698 .and(
699 function::Column::ArgTypes
700 .eq(DataTypeArray::from(pb_function.arg_types.clone())),
701 ),
702 )
703 .count(db)
704 .await?;
705 if count > 0 {
706 assert_eq!(count, 1);
707 return Err(MetaError::catalog_duplicated("function", &pb_function.name));
708 }
709 Ok(())
710}
711
712pub async fn check_connection_name_duplicate<C>(
714 pb_connection: &PbConnection,
715 db: &C,
716) -> MetaResult<()>
717where
718 C: ConnectionTrait,
719{
720 let count = Connection::find()
721 .inner_join(Object)
722 .filter(
723 object::Column::DatabaseId
724 .eq(pb_connection.database_id)
725 .and(object::Column::SchemaId.eq(pb_connection.schema_id))
726 .and(connection::Column::Name.eq(&pb_connection.name)),
727 )
728 .count(db)
729 .await?;
730 if count > 0 {
731 assert_eq!(count, 1);
732 return Err(MetaError::catalog_duplicated(
733 "connection",
734 &pb_connection.name,
735 ));
736 }
737 Ok(())
738}
739
740pub async fn check_secret_name_duplicate<C>(pb_secret: &PbSecret, db: &C) -> MetaResult<()>
741where
742 C: ConnectionTrait,
743{
744 let count = Secret::find()
745 .inner_join(Object)
746 .filter(
747 object::Column::DatabaseId
748 .eq(pb_secret.database_id)
749 .and(object::Column::SchemaId.eq(pb_secret.schema_id))
750 .and(secret::Column::Name.eq(&pb_secret.name)),
751 )
752 .count(db)
753 .await?;
754 if count > 0 {
755 assert_eq!(count, 1);
756 return Err(MetaError::catalog_duplicated("secret", &pb_secret.name));
757 }
758 Ok(())
759}
760
761pub async fn check_subscription_name_duplicate<C>(
762 pb_subscription: &PbSubscription,
763 db: &C,
764) -> MetaResult<()>
765where
766 C: ConnectionTrait,
767{
768 let count = Subscription::find()
769 .inner_join(Object)
770 .filter(
771 object::Column::DatabaseId
772 .eq(pb_subscription.database_id)
773 .and(object::Column::SchemaId.eq(pb_subscription.schema_id))
774 .and(subscription::Column::Name.eq(&pb_subscription.name)),
775 )
776 .count(db)
777 .await?;
778 if count > 0 {
779 assert_eq!(count, 1);
780 return Err(MetaError::catalog_duplicated(
781 "subscription",
782 &pb_subscription.name,
783 ));
784 }
785 Ok(())
786}
787
788pub async fn check_user_name_duplicate<C>(name: &str, db: &C) -> MetaResult<()>
790where
791 C: ConnectionTrait,
792{
793 let count = User::find()
794 .filter(user::Column::Name.eq(name))
795 .count(db)
796 .await?;
797 if count > 0 {
798 assert_eq!(count, 1);
799 return Err(MetaError::catalog_duplicated("user", name));
800 }
801 Ok(())
802}
803
804pub async fn check_relation_name_duplicate<C>(
806 name: &str,
807 database_id: DatabaseId,
808 schema_id: SchemaId,
809 db: &C,
810) -> MetaResult<()>
811where
812 C: ConnectionTrait,
813{
814 macro_rules! check_duplicated {
815 ($obj_type:expr, $entity:ident, $table:ident) => {
816 let object_id = Object::find()
817 .select_only()
818 .column(object::Column::Oid)
819 .inner_join($entity)
820 .filter(
821 object::Column::DatabaseId
822 .eq(Some(database_id))
823 .and(object::Column::SchemaId.eq(Some(schema_id)))
824 .and($table::Column::Name.eq(name)),
825 )
826 .into_tuple::<ObjectId>()
827 .one(db)
828 .await?;
829 if let Some(oid) = object_id {
830 let check_creation = if $obj_type == ObjectType::View {
831 false
832 } else if $obj_type == ObjectType::Source {
833 let source_info = Source::find_by_id(oid.as_source_id())
834 .select_only()
835 .column(source::Column::SourceInfo)
836 .into_tuple::<Option<StreamSourceInfo>>()
837 .one(db)
838 .await?
839 .unwrap();
840 source_info.map_or(false, |info| info.to_protobuf().is_shared())
841 } else {
842 true
843 };
844 let job_id = oid.as_job_id();
845 return if check_creation
846 && !matches!(
847 StreamingJob::find_by_id(job_id)
848 .select_only()
849 .column(streaming_job::Column::JobStatus)
850 .into_tuple::<JobStatus>()
851 .one(db)
852 .await?,
853 Some(JobStatus::Created)
854 ) {
855 Err(MetaError::catalog_under_creation(
856 $obj_type.as_str(),
857 name,
858 job_id,
859 ))
860 } else {
861 Err(MetaError::catalog_duplicated($obj_type.as_str(), name))
862 };
863 }
864 };
865 }
866 check_duplicated!(ObjectType::Table, Table, table);
867 check_duplicated!(ObjectType::Source, Source, source);
868 check_duplicated!(ObjectType::Sink, Sink, sink);
869 check_duplicated!(ObjectType::Index, Index, index);
870 check_duplicated!(ObjectType::View, View, view);
871
872 Ok(())
873}
874
875pub async fn check_schema_name_duplicate<C>(
877 name: &str,
878 database_id: DatabaseId,
879 db: &C,
880) -> MetaResult<()>
881where
882 C: ConnectionTrait,
883{
884 let count = Object::find()
885 .inner_join(Schema)
886 .filter(
887 object::Column::ObjType
888 .eq(ObjectType::Schema)
889 .and(object::Column::DatabaseId.eq(Some(database_id)))
890 .and(schema::Column::Name.eq(name)),
891 )
892 .count(db)
893 .await?;
894 if count != 0 {
895 return Err(MetaError::catalog_duplicated("schema", name));
896 }
897
898 Ok(())
899}
900
901pub async fn validate_restrict_drop_and_collect_owned_objects<C>(
908 object_type: ObjectType,
909 object_id: ObjectId,
910 db: &C,
911) -> MetaResult<Vec<PartialObject>>
912where
913 C: ConnectionTrait,
914{
915 let referring_objects = get_referring_objects(object_id, object_type, db).await?;
916 let owned_object_ids = get_belong_objects(db, object_id)
917 .await?
918 .into_iter()
919 .map(|object| object.oid)
920 .collect::<HashSet<_>>();
921 let mut non_owned_objects = referring_objects.clone();
922 non_owned_objects.retain(|object| !owned_object_ids.contains(&object.oid));
923 if object_type == ObjectType::Table {
924 non_owned_objects.retain(|object| object.obj_type != ObjectType::Index);
925 }
926
927 if !non_owned_objects.is_empty() {
928 let referring_objs_map = non_owned_objects
929 .into_iter()
930 .into_group_map_by(|o| o.obj_type);
931 let mut details = vec![];
932 for (obj_type, objs) in referring_objs_map {
933 match obj_type {
934 ObjectType::Table => {
935 let tables: Vec<(String, String)> = Object::find()
936 .join(JoinType::InnerJoin, object::Relation::Table.def())
937 .join(JoinType::InnerJoin, object::Relation::Database2.def())
938 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
939 .select_only()
940 .column(schema::Column::Name)
941 .column(table::Column::Name)
942 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
943 .into_tuple()
944 .all(db)
945 .await?;
946 details.extend(tables.into_iter().map(|(schema_name, table_name)| {
947 format!(
948 "materialized view {}.{} depends on it",
949 schema_name, table_name
950 )
951 }));
952 }
953 ObjectType::Sink => {
954 let sinks: Vec<(String, String)> = Object::find()
955 .join(JoinType::InnerJoin, object::Relation::Sink.def())
956 .join(JoinType::InnerJoin, object::Relation::Database2.def())
957 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
958 .select_only()
959 .column(schema::Column::Name)
960 .column(sink::Column::Name)
961 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
962 .into_tuple()
963 .all(db)
964 .await?;
965 details.extend(sinks.into_iter().map(|(schema_name, sink_name)| {
966 format!("sink {}.{} depends on it", schema_name, sink_name)
967 }));
968 }
969 ObjectType::View => {
970 let views: Vec<(String, String)> = Object::find()
971 .join(JoinType::InnerJoin, object::Relation::View.def())
972 .join(JoinType::InnerJoin, object::Relation::Database2.def())
973 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
974 .select_only()
975 .column(schema::Column::Name)
976 .column(view::Column::Name)
977 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
978 .into_tuple()
979 .all(db)
980 .await?;
981 details.extend(views.into_iter().map(|(schema_name, view_name)| {
982 format!("view {}.{} depends on it", schema_name, view_name)
983 }));
984 }
985 ObjectType::Subscription => {
986 let subscriptions: Vec<(String, String)> = Object::find()
987 .join(JoinType::InnerJoin, object::Relation::Subscription.def())
988 .join(JoinType::InnerJoin, object::Relation::Database2.def())
989 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
990 .select_only()
991 .column(schema::Column::Name)
992 .column(subscription::Column::Name)
993 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
994 .into_tuple()
995 .all(db)
996 .await?;
997 details.extend(subscriptions.into_iter().map(
998 |(schema_name, subscription_name)| {
999 format!(
1000 "subscription {}.{} depends on it",
1001 schema_name, subscription_name
1002 )
1003 },
1004 ));
1005 }
1006 ObjectType::Source => {
1007 let sources: Vec<(String, String)> = Object::find()
1008 .join(JoinType::InnerJoin, object::Relation::Source.def())
1009 .join(JoinType::InnerJoin, object::Relation::Database2.def())
1010 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
1011 .select_only()
1012 .column(schema::Column::Name)
1013 .column(source::Column::Name)
1014 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
1015 .into_tuple()
1016 .all(db)
1017 .await?;
1018 details.extend(sources.into_iter().map(|(schema_name, view_name)| {
1019 format!("source {}.{} depends on it", schema_name, view_name)
1020 }));
1021 }
1022 ObjectType::Connection => {
1023 let connections: Vec<(String, String)> = Object::find()
1024 .join(JoinType::InnerJoin, object::Relation::Connection.def())
1025 .join(JoinType::InnerJoin, object::Relation::Database2.def())
1026 .join(JoinType::InnerJoin, object::Relation::Schema2.def())
1027 .select_only()
1028 .column(schema::Column::Name)
1029 .column(connection::Column::Name)
1030 .filter(object::Column::Oid.is_in(objs.iter().map(|o| o.oid)))
1031 .into_tuple()
1032 .all(db)
1033 .await?;
1034 details.extend(connections.into_iter().map(|(schema_name, view_name)| {
1035 format!("connection {}.{} depends on it", schema_name, view_name)
1036 }));
1037 }
1038 _ => bail!("unexpected referring object type: {}", obj_type.as_str()),
1040 }
1041 }
1042 if details.is_empty() {
1043 return Ok(referring_objects);
1044 }
1045
1046 return Err(MetaError::permission_denied(format!(
1047 "{} used by {} other objects. \nDETAIL: {}\n\
1048 {}",
1049 object_type.as_str(),
1050 details.len(),
1051 details.join("\n"),
1052 match object_type {
1053 ObjectType::Function | ObjectType::Connection | ObjectType::Secret =>
1054 "HINT: DROP the dependent objects first.",
1055 ObjectType::Database | ObjectType::Schema => unreachable!(),
1056 _ => "HINT: Use DROP ... CASCADE to drop the dependent objects too.",
1057 }
1058 )));
1059 }
1060 Ok(referring_objects)
1061}
1062
1063async fn get_incoming_sink_objects_for_target<C>(
1064 target_id: ObjectId,
1065 target_type: ObjectType,
1066 db: &C,
1067) -> MetaResult<Vec<PartialObject>>
1068where
1069 C: ConnectionTrait,
1070{
1071 match target_type {
1072 ObjectType::Table => {
1073 let incoming_sink_ids = Sink::find()
1074 .select_only()
1075 .column(sink::Column::SinkId)
1076 .filter(sink::Column::TargetTable.eq(target_id.as_table_id()))
1077 .into_tuple::<SinkId>()
1078 .all(db)
1079 .await?;
1080 Object::find()
1081 .filter(object::Column::Oid.is_in(incoming_sink_ids))
1082 .into_partial_model()
1083 .all(db)
1084 .await
1085 .map_err(Into::into)
1086 }
1087 _ => Ok(vec![]),
1088 }
1089}
1090
1091pub async fn get_referring_objects<C>(
1093 object_id: ObjectId,
1094 object_type: ObjectType,
1095 db: &C,
1096) -> MetaResult<Vec<PartialObject>>
1097where
1098 C: ConnectionTrait,
1099{
1100 let mut objects: Vec<PartialObject> = ObjectDependency::find()
1101 .filter(object_dependency::Column::Oid.eq(object_id))
1102 .join(
1103 JoinType::InnerJoin,
1104 object_dependency::Relation::Object1.def(),
1105 )
1106 .into_partial_model()
1107 .all(db)
1108 .await?;
1109
1110 let incoming_sink_objects =
1111 get_incoming_sink_objects_for_target(object_id, object_type, db).await?;
1112 let mut existing_object_ids: HashSet<ObjectId> =
1113 objects.iter().map(|object| object.oid).collect();
1114 objects.extend(
1115 incoming_sink_objects
1116 .into_iter()
1117 .filter(|object| existing_object_ids.insert(object.oid)),
1118 );
1119
1120 Ok(objects)
1121}
1122
1123pub async fn ensure_schema_empty<C>(schema_id: SchemaId, db: &C) -> MetaResult<()>
1125where
1126 C: ConnectionTrait,
1127{
1128 let count = Object::find()
1129 .filter(object::Column::SchemaId.eq(Some(schema_id)))
1130 .count(db)
1131 .await?;
1132 if count != 0 {
1133 return Err(MetaError::permission_denied("schema is not empty"));
1134 }
1135
1136 Ok(())
1137}
1138
1139pub async fn list_user_info_by_ids<C>(
1141 user_ids: impl IntoIterator<Item = UserId>,
1142 db: &C,
1143) -> MetaResult<Vec<PbUserInfo>>
1144where
1145 C: ConnectionTrait,
1146{
1147 let mut user_infos = vec![];
1148 for user_id in user_ids {
1149 let user = User::find_by_id(user_id)
1150 .one(db)
1151 .await?
1152 .ok_or_else(|| MetaError::catalog_id_not_found("user", user_id))?;
1153 let mut user_info: PbUserInfo = user.into();
1154 user_info.grant_privileges = get_user_privilege(user_id, db).await?;
1155 user_infos.push(user_info);
1156 }
1157 Ok(user_infos)
1158}
1159
1160pub async fn get_object_owner<C>(object_id: ObjectId, db: &C) -> MetaResult<UserId>
1162where
1163 C: ConnectionTrait,
1164{
1165 let obj_owner: UserId = Object::find_by_id(object_id)
1166 .select_only()
1167 .column(object::Column::OwnerId)
1168 .into_tuple()
1169 .one(db)
1170 .await?
1171 .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1172 Ok(obj_owner)
1173}
1174
1175pub fn construct_privilege_dependency_query(ids: Vec<PrivilegeId>) -> WithQuery {
1200 let cte_alias = Alias::new("granted_privilege_ids");
1201 let cte_return_privilege_alias = Alias::new("id");
1202 let cte_return_user_alias = Alias::new("user_id");
1203
1204 let mut base_query = SelectStatement::new()
1205 .columns([user_privilege::Column::Id, user_privilege::Column::UserId])
1206 .from(UserPrivilege)
1207 .and_where(user_privilege::Column::Id.is_in(ids))
1208 .to_owned();
1209
1210 let cte_referencing = Query::select()
1211 .columns([
1212 (UserPrivilege, user_privilege::Column::Id),
1213 (UserPrivilege, user_privilege::Column::UserId),
1214 ])
1215 .from(UserPrivilege)
1216 .inner_join(
1217 cte_alias.clone(),
1218 Expr::col((cte_alias.clone(), cte_return_privilege_alias.clone()))
1219 .equals(user_privilege::Column::DependentId),
1220 )
1221 .to_owned();
1222
1223 let mut common_table_expr = CommonTableExpression::new();
1224 common_table_expr
1225 .query(base_query.union(UnionType::All, cte_referencing).to_owned())
1226 .columns([
1227 cte_return_privilege_alias.clone(),
1228 cte_return_user_alias.clone(),
1229 ])
1230 .table_name(cte_alias.clone());
1231
1232 SelectStatement::new()
1233 .columns([cte_return_privilege_alias, cte_return_user_alias])
1234 .from(cte_alias)
1235 .to_owned()
1236 .with(
1237 WithClause::new()
1238 .recursive(true)
1239 .cte(common_table_expr)
1240 .to_owned(),
1241 )
1242}
1243
1244pub async fn get_internal_tables_by_id<C>(job_id: JobId, db: &C) -> MetaResult<Vec<TableId>>
1245where
1246 C: ConnectionTrait,
1247{
1248 let table_ids: Vec<TableId> = Table::find()
1249 .select_only()
1250 .column(table::Column::TableId)
1251 .filter(
1252 table::Column::TableType
1253 .eq(TableType::Internal)
1254 .and(table::Column::BelongsToJobId.eq(job_id)),
1255 )
1256 .into_tuple()
1257 .all(db)
1258 .await?;
1259 Ok(table_ids)
1260}
1261
1262pub async fn get_index_state_tables_by_table_id<C>(
1263 table_id: TableId,
1264 db: &C,
1265) -> MetaResult<Vec<TableId>>
1266where
1267 C: ConnectionTrait,
1268{
1269 let mut index_table_ids: Vec<TableId> = Index::find()
1270 .select_only()
1271 .column(index::Column::IndexTableId)
1272 .filter(index::Column::PrimaryTableId.eq(table_id))
1273 .into_tuple()
1274 .all(db)
1275 .await?;
1276
1277 if !index_table_ids.is_empty() {
1278 let internal_table_ids: Vec<TableId> = Table::find()
1279 .select_only()
1280 .column(table::Column::TableId)
1281 .filter(
1282 table::Column::TableType
1283 .eq(TableType::Internal)
1284 .and(table::Column::BelongsToJobId.is_in(index_table_ids.clone())),
1285 )
1286 .into_tuple()
1287 .all(db)
1288 .await?;
1289
1290 index_table_ids.extend(internal_table_ids);
1291 }
1292
1293 Ok(index_table_ids)
1294}
1295
1296pub async fn get_iceberg_related_object_ids<C>(
1298 object_id: ObjectId,
1299 db: &C,
1300) -> MetaResult<Vec<ObjectId>>
1301where
1302 C: ConnectionTrait,
1303{
1304 let object = Object::find_by_id(object_id)
1305 .one(db)
1306 .await?
1307 .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1308 if object.obj_type != ObjectType::Table {
1309 return Ok(vec![]);
1310 }
1311
1312 let table = Table::find_by_id(object_id.as_table_id())
1313 .one(db)
1314 .await?
1315 .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
1316 if !matches!(table.engine, Some(table::Engine::Iceberg)) {
1317 return Ok(vec![]);
1318 }
1319
1320 let database_id = object.database_id.unwrap();
1321 let schema_id = object.schema_id.unwrap();
1322
1323 let mut related_objects = vec![];
1324
1325 let iceberg_sink_name = format!("{}{}", ICEBERG_SINK_PREFIX, table.name);
1326 let iceberg_sink_id = Sink::find()
1327 .inner_join(Object)
1328 .select_only()
1329 .column(sink::Column::SinkId)
1330 .filter(
1331 object::Column::DatabaseId
1332 .eq(database_id)
1333 .and(object::Column::SchemaId.eq(schema_id))
1334 .and(sink::Column::Name.eq(&iceberg_sink_name)),
1335 )
1336 .into_tuple::<SinkId>()
1337 .one(db)
1338 .await?;
1339 if let Some(sink_id) = iceberg_sink_id {
1340 related_objects.push(sink_id.as_object_id());
1341 let sink_internal_tables = get_internal_tables_by_id(sink_id.as_job_id(), db).await?;
1342 related_objects.extend(
1343 sink_internal_tables
1344 .into_iter()
1345 .map(|tid| tid.as_object_id()),
1346 );
1347 } else {
1348 warn!(
1349 "iceberg table {} missing sink {}",
1350 table.name, iceberg_sink_name
1351 );
1352 }
1353
1354 let iceberg_source_name = format!("{}{}", ICEBERG_SOURCE_PREFIX, table.name);
1355 let iceberg_source_id = Source::find()
1356 .inner_join(Object)
1357 .select_only()
1358 .column(source::Column::SourceId)
1359 .filter(
1360 object::Column::DatabaseId
1361 .eq(database_id)
1362 .and(object::Column::SchemaId.eq(schema_id))
1363 .and(source::Column::Name.eq(&iceberg_source_name)),
1364 )
1365 .into_tuple::<SourceId>()
1366 .one(db)
1367 .await?;
1368 if let Some(source_id) = iceberg_source_id {
1369 related_objects.push(source_id.as_object_id());
1370 } else {
1371 warn!(
1372 "iceberg table {} missing source {}",
1373 table.name, iceberg_source_name
1374 );
1375 }
1376
1377 Ok(related_objects)
1378}
1379
1380pub(crate) async fn load_streaming_jobs_by_ids<C>(
1382 txn: &C,
1383 job_ids: impl IntoIterator<Item = JobId>,
1384) -> MetaResult<HashMap<JobId, streaming_job::Model>>
1385where
1386 C: ConnectionTrait,
1387{
1388 let job_ids: HashSet<JobId> = job_ids.into_iter().collect();
1389 if job_ids.is_empty() {
1390 return Ok(HashMap::new());
1391 }
1392 let jobs = streaming_job::Entity::find()
1393 .filter(streaming_job::Column::JobId.is_in(job_ids.clone()))
1394 .all(txn)
1395 .await?;
1396 Ok(jobs.into_iter().map(|job| (job.job_id, job)).collect())
1397}
1398
1399#[derive(Clone, DerivePartialModel, FromQueryResult)]
1400#[sea_orm(entity = "UserPrivilege")]
1401pub struct PartialUserPrivilege {
1402 pub id: PrivilegeId,
1403 pub user_id: UserId,
1404}
1405
1406pub async fn get_referring_privileges_cascade<C>(
1407 ids: Vec<PrivilegeId>,
1408 db: &C,
1409) -> MetaResult<Vec<PartialUserPrivilege>>
1410where
1411 C: ConnectionTrait,
1412{
1413 let query = construct_privilege_dependency_query(ids);
1414 let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder());
1415 let privileges = PartialUserPrivilege::find_by_statement(Statement::from_sql_and_values(
1416 db.get_database_backend(),
1417 sql,
1418 values,
1419 ))
1420 .all(db)
1421 .await?;
1422
1423 Ok(privileges)
1424}
1425
1426pub async fn ensure_privileges_not_referred<C>(ids: Vec<PrivilegeId>, db: &C) -> MetaResult<()>
1428where
1429 C: ConnectionTrait,
1430{
1431 let count = UserPrivilege::find()
1432 .filter(user_privilege::Column::DependentId.is_in(ids))
1433 .count(db)
1434 .await?;
1435 if count != 0 {
1436 return Err(MetaError::permission_denied(format!(
1437 "privileges granted to {} other ones.",
1438 count
1439 )));
1440 }
1441 Ok(())
1442}
1443
1444pub async fn get_user_privilege<C>(user_id: UserId, db: &C) -> MetaResult<Vec<PbGrantPrivilege>>
1446where
1447 C: ConnectionTrait,
1448{
1449 let user_privileges = UserPrivilege::find()
1450 .find_also_related(Object)
1451 .filter(user_privilege::Column::UserId.eq(user_id))
1452 .all(db)
1453 .await?;
1454 Ok(user_privileges
1455 .into_iter()
1456 .map(|(privilege, object)| {
1457 let object = object.unwrap();
1458 let obj = match object.obj_type {
1459 ObjectType::Database => PbGrantObject::DatabaseId(object.oid.as_database_id()),
1460 ObjectType::Schema => PbGrantObject::SchemaId(object.oid.as_schema_id()),
1461 ObjectType::Table | ObjectType::Index => {
1462 PbGrantObject::TableId(object.oid.as_table_id())
1463 }
1464 ObjectType::Source => PbGrantObject::SourceId(object.oid.as_source_id()),
1465 ObjectType::Sink => PbGrantObject::SinkId(object.oid.as_sink_id()),
1466 ObjectType::View => PbGrantObject::ViewId(object.oid.as_view_id()),
1467 ObjectType::Function => PbGrantObject::FunctionId(object.oid.as_function_id()),
1468 ObjectType::Connection => {
1469 PbGrantObject::ConnectionId(object.oid.as_connection_id())
1470 }
1471 ObjectType::Subscription => {
1472 PbGrantObject::SubscriptionId(object.oid.as_subscription_id())
1473 }
1474 ObjectType::Secret => PbGrantObject::SecretId(object.oid.as_secret_id()),
1475 };
1476 PbGrantPrivilege {
1477 action_with_opts: vec![PbActionWithGrantOption {
1478 action: PbAction::from(privilege.action) as _,
1479 with_grant_option: privilege.with_grant_option,
1480 granted_by: privilege.granted_by as _,
1481 }],
1482 object: Some(obj),
1483 }
1484 })
1485 .collect())
1486}
1487
1488pub(crate) async fn upsert_user_privileges<C>(
1491 db: &C,
1492 privileges: impl IntoIterator<Item = user_privilege::ActiveModel>,
1493) -> MetaResult<()>
1494where
1495 C: ConnectionTrait,
1496{
1497 for privilege in privileges {
1498 let mut on_conflict = OnConflict::columns([
1499 user_privilege::Column::UserId,
1500 user_privilege::Column::Oid,
1501 user_privilege::Column::Action,
1502 user_privilege::Column::GrantedBy,
1503 ]);
1504 if *privilege.with_grant_option.as_ref() {
1505 on_conflict.update_column(user_privilege::Column::WithGrantOption);
1506 } else {
1507 on_conflict.update_column(user_privilege::Column::UserId);
1509 }
1510
1511 UserPrivilege::insert(privilege)
1512 .on_conflict(on_conflict)
1513 .do_nothing()
1514 .exec(db)
1515 .await?;
1516 }
1517 Ok(())
1518}
1519
1520pub async fn get_table_columns(
1521 txn: &impl ConnectionTrait,
1522 id: TableId,
1523) -> MetaResult<ColumnCatalogArray> {
1524 let columns = Table::find_by_id(id)
1525 .select_only()
1526 .columns([table::Column::Columns])
1527 .into_tuple::<ColumnCatalogArray>()
1528 .one(txn)
1529 .await?
1530 .ok_or_else(|| MetaError::catalog_id_not_found("table", id))?;
1531 Ok(columns)
1532}
1533
1534pub async fn grant_default_privileges_automatically<C>(
1537 db: &C,
1538 object_id: impl Into<ObjectId>,
1539) -> MetaResult<Vec<PbUserInfo>>
1540where
1541 C: ConnectionTrait,
1542{
1543 let object_id = object_id.into();
1544 let object = Object::find_by_id(object_id)
1545 .one(db)
1546 .await?
1547 .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
1548 assert_ne!(object.obj_type, ObjectType::Database);
1549
1550 let for_mview_filter = if object.obj_type == ObjectType::Table {
1551 let table_type = Table::find_by_id(object_id.as_table_id())
1552 .select_only()
1553 .column(table::Column::TableType)
1554 .into_tuple::<TableType>()
1555 .one(db)
1556 .await?
1557 .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
1558 user_default_privilege::Column::ForMaterializedView
1559 .eq(table_type == TableType::MaterializedView)
1560 } else {
1561 user_default_privilege::Column::ForMaterializedView.eq(false)
1562 };
1563 let schema_filter = if let Some(schema_id) = &object.schema_id {
1564 user_default_privilege::Column::SchemaId.eq(*schema_id)
1565 } else {
1566 user_default_privilege::Column::SchemaId.is_null()
1567 };
1568
1569 let default_privileges: Vec<(UserId, UserId, Action, bool)> = UserDefaultPrivilege::find()
1570 .select_only()
1571 .columns([
1572 user_default_privilege::Column::Grantee,
1573 user_default_privilege::Column::GrantedBy,
1574 user_default_privilege::Column::Action,
1575 user_default_privilege::Column::WithGrantOption,
1576 ])
1577 .filter(
1578 user_default_privilege::Column::DatabaseId
1579 .eq(object.database_id.unwrap())
1580 .and(schema_filter)
1581 .and(user_default_privilege::Column::UserId.eq(object.owner_id))
1582 .and(user_default_privilege::Column::ObjectType.eq(object.obj_type))
1583 .and(for_mview_filter),
1584 )
1585 .into_tuple()
1586 .all(db)
1587 .await?;
1588 if default_privileges.is_empty() {
1589 return Ok(vec![]);
1590 }
1591
1592 let updated_user_ids = default_privileges
1593 .iter()
1594 .map(|(grantee, _, _, _)| *grantee)
1595 .collect::<HashSet<_>>();
1596
1597 let mut new_privileges = vec![];
1598 for (grantee, granted_by, action, with_grant_option) in default_privileges {
1599 new_privileges.push(user_privilege::ActiveModel {
1600 user_id: Set(grantee),
1601 oid: Set(object_id),
1602 granted_by: Set(granted_by),
1603 action: Set(action),
1604 with_grant_option: Set(with_grant_option),
1605 ..Default::default()
1606 });
1607 if action == Action::Select {
1608 let internal_table_ids = get_internal_tables_by_id(object_id.as_job_id(), db).await?;
1610 new_privileges.extend(internal_table_ids.into_iter().map(|internal_table_id| {
1611 user_privilege::ActiveModel {
1612 user_id: Set(grantee),
1613 oid: Set(internal_table_id.as_object_id()),
1614 granted_by: Set(granted_by),
1615 action: Set(Action::Select),
1616 with_grant_option: Set(with_grant_option),
1617 ..Default::default()
1618 }
1619 }));
1620
1621 let iceberg_privilege_object_ids =
1623 get_iceberg_related_object_ids(object_id, db).await?;
1624 new_privileges.extend(iceberg_privilege_object_ids.into_iter().map(
1625 |iceberg_object_id| user_privilege::ActiveModel {
1626 user_id: Set(grantee),
1627 oid: Set(iceberg_object_id),
1628 granted_by: Set(granted_by),
1629 action: Set(action),
1630 with_grant_option: Set(with_grant_option),
1631 ..Default::default()
1632 },
1633 ));
1634 }
1635 }
1636 upsert_user_privileges(db, new_privileges).await?;
1637
1638 let updated_user_infos = list_user_info_by_ids(updated_user_ids, db).await?;
1639 Ok(updated_user_infos)
1640}
1641
1642pub fn extract_grant_obj_id(object: &PbGrantObject) -> ObjectId {
1644 match object {
1645 PbGrantObject::DatabaseId(id) => (*id).into(),
1646 PbGrantObject::SchemaId(id) => (*id).into(),
1647 PbGrantObject::TableId(id) => (*id).into(),
1648 PbGrantObject::SourceId(id) => (*id).into(),
1649 PbGrantObject::SinkId(id) => (*id).into(),
1650 PbGrantObject::ViewId(id) => (*id).into(),
1651 PbGrantObject::FunctionId(id) => (*id).into(),
1652 PbGrantObject::SubscriptionId(id) => (*id).into(),
1653 PbGrantObject::ConnectionId(id) => (*id).into(),
1654 PbGrantObject::SecretId(id) => (*id).into(),
1655 }
1656}
1657
1658pub async fn insert_fragment_relations(
1659 db: &impl ConnectionTrait,
1660 downstream_fragment_relations: &FragmentDownstreamRelation,
1661) -> MetaResult<()> {
1662 let mut relations = vec![];
1663 for (upstream_fragment_id, downstreams) in downstream_fragment_relations {
1664 for downstream in downstreams {
1665 relations.push(
1666 fragment_relation::Model {
1667 source_fragment_id: *upstream_fragment_id as _,
1668 target_fragment_id: downstream.downstream_fragment_id as _,
1669 dispatcher_type: downstream.dispatcher_type,
1670 dist_key_indices: downstream
1671 .dist_key_indices
1672 .iter()
1673 .map(|idx| *idx as i32)
1674 .collect_vec()
1675 .into(),
1676 output_indices: downstream
1677 .output_mapping
1678 .indices
1679 .iter()
1680 .map(|idx| *idx as i32)
1681 .collect_vec()
1682 .into(),
1683 output_type_mapping: Some(downstream.output_mapping.types.clone().into()),
1684 }
1685 .into_active_model(),
1686 );
1687 }
1688 }
1689 if !relations.is_empty() {
1690 FragmentRelation::insert_many(relations).exec(db).await?;
1691 }
1692 Ok(())
1693}
1694
1695pub fn compose_dispatchers(
1696 source_fragment_distribution: DistributionType,
1697 source_fragment_actors: &HashMap<crate::model::ActorId, Option<Bitmap>>,
1698 target_fragment_id: crate::model::FragmentId,
1699 target_fragment_distribution: DistributionType,
1700 target_fragment_actors: &HashMap<crate::model::ActorId, Option<Bitmap>>,
1701 dispatcher_type: DispatcherType,
1702 dist_key_indices: Vec<u32>,
1703 output_mapping: PbDispatchOutputMapping,
1704) -> (
1705 HashMap<crate::model::ActorId, PbDispatcher>,
1706 Option<HashMap<crate::model::ActorId, crate::model::ActorId>>,
1707) {
1708 match dispatcher_type {
1709 DispatcherType::Hash => {
1710 let dispatcher = PbDispatcher {
1711 r#type: PbDispatcherType::from(dispatcher_type) as _,
1712 dist_key_indices,
1713 output_mapping: output_mapping.into(),
1714 hash_mapping: Some(
1715 ActorMapping::from_bitmaps(
1716 &target_fragment_actors
1717 .iter()
1718 .map(|(actor_id, bitmap)| {
1719 (
1720 *actor_id as _,
1721 bitmap
1722 .clone()
1723 .expect("downstream hash dispatch must have distribution"),
1724 )
1725 })
1726 .collect(),
1727 )
1728 .to_protobuf(),
1729 ),
1730 dispatcher_id: target_fragment_id,
1731 downstream_actor_id: target_fragment_actors.keys().copied().collect(),
1732 };
1733 (
1734 source_fragment_actors
1735 .keys()
1736 .map(|source_actor_id| (*source_actor_id, dispatcher.clone()))
1737 .collect(),
1738 None,
1739 )
1740 }
1741 DispatcherType::Broadcast | DispatcherType::Simple => {
1742 let dispatcher = PbDispatcher {
1743 r#type: PbDispatcherType::from(dispatcher_type) as _,
1744 dist_key_indices,
1745 output_mapping: output_mapping.into(),
1746 hash_mapping: None,
1747 dispatcher_id: target_fragment_id,
1748 downstream_actor_id: target_fragment_actors.keys().copied().collect(),
1749 };
1750 (
1751 source_fragment_actors
1752 .keys()
1753 .map(|source_actor_id| (*source_actor_id, dispatcher.clone()))
1754 .collect(),
1755 None,
1756 )
1757 }
1758 DispatcherType::NoShuffle => {
1759 let no_shuffle_map = resolve_no_shuffle_actor_mapping(
1760 source_fragment_distribution,
1761 source_fragment_actors
1762 .iter()
1763 .map(|(&id, bitmap)| (id, bitmap)),
1764 target_fragment_distribution,
1765 target_fragment_actors
1766 .iter()
1767 .map(|(&id, bitmap)| (id, bitmap)),
1768 );
1769 let dispatchers = no_shuffle_map
1770 .iter()
1771 .map(|(&upstream_actor_id, &downstream_actor_id)| {
1772 (
1773 upstream_actor_id,
1774 PbDispatcher {
1775 r#type: PbDispatcherType::NoShuffle as _,
1776 dist_key_indices: dist_key_indices.clone(),
1777 output_mapping: output_mapping.clone().into(),
1778 hash_mapping: None,
1779 dispatcher_id: target_fragment_id,
1780 downstream_actor_id: vec![downstream_actor_id],
1781 },
1782 )
1783 })
1784 .collect();
1785 (dispatchers, Some(no_shuffle_map))
1786 }
1787 }
1788}
1789
1790pub fn resolve_no_shuffle_actor_mapping<
1800 'a,
1801 ActorId: Copy + Eq + std::hash::Hash + std::fmt::Debug,
1802>(
1803 source_fragment_distribution: DistributionType,
1804 source_fragment_actors: impl IntoIterator<Item = (ActorId, &'a Option<Bitmap>)>,
1805 target_fragment_distribution: DistributionType,
1806 target_fragment_actors: impl IntoIterator<Item = (ActorId, &'a Option<Bitmap>)>,
1807) -> HashMap<ActorId, ActorId> {
1808 assert_eq!(source_fragment_distribution, target_fragment_distribution);
1809
1810 match source_fragment_distribution {
1811 DistributionType::Single => {
1812 let assert_singleton = |bitmap: &Option<Bitmap>| {
1813 assert!(
1814 bitmap.as_ref().map(|bitmap| bitmap.all()).unwrap_or(true),
1815 "not singleton: {:?}",
1816 bitmap
1817 );
1818 };
1819 let (source_actor_id, bitmap) =
1820 Itertools::exactly_one(source_fragment_actors.into_iter())
1821 .ok()
1822 .expect("Single distribution should have exactly one source actor");
1823 assert_singleton(bitmap);
1824 let (target_actor_id, bitmap) =
1825 Itertools::exactly_one(target_fragment_actors.into_iter())
1826 .ok()
1827 .expect("Single distribution should have exactly one target actor");
1828 assert_singleton(bitmap);
1829 HashMap::from([(source_actor_id, target_actor_id)])
1830 }
1831 DistributionType::Hash => {
1832 let mut target_by_vnode: HashMap<_, _> = target_fragment_actors
1834 .into_iter()
1835 .map(|(actor_id, bitmap)| {
1836 let bitmap = bitmap
1837 .as_ref()
1838 .expect("hash distribution should have bitmap");
1839 let first_vnode = bitmap.iter_vnodes().next().expect("non-empty bitmap");
1840 (first_vnode, (actor_id, bitmap))
1841 })
1842 .collect();
1843
1844 let target_count = target_by_vnode.len();
1845
1846 let mapping: HashMap<_, _> = source_fragment_actors
1848 .into_iter()
1849 .map(|(source_actor_id, source_bitmap)| {
1850 let source_bitmap = source_bitmap
1851 .as_ref()
1852 .expect("hash distribution should have bitmap");
1853 let first_vnode = source_bitmap
1854 .iter_vnodes()
1855 .next()
1856 .expect("non-empty bitmap");
1857 let (target_actor_id, target_bitmap) =
1858 target_by_vnode.remove(&first_vnode).unwrap_or_else(|| {
1859 panic!(
1860 "cannot find matched target actor: {:?} first_vnode {:?}",
1861 source_actor_id, first_vnode,
1862 )
1863 });
1864 assert_eq!(
1865 source_bitmap, target_bitmap,
1866 "bitmap mismatch for source {:?} target {:?} at first_vnode {:?}",
1867 source_actor_id, target_actor_id, first_vnode,
1868 );
1869 (source_actor_id, target_actor_id)
1870 })
1871 .collect();
1872
1873 assert_eq!(
1874 mapping.len(),
1875 target_count,
1876 "no-shuffle should have equal upstream downstream actor count: {} vs {}",
1877 mapping.len(),
1878 target_count,
1879 );
1880
1881 mapping
1882 }
1883 }
1884}
1885
1886pub fn rebuild_fragment_mapping(fragment: &SharedFragmentInfo) -> PbFragmentWorkerSlotMapping {
1887 let fragment_worker_slot_mapping = match fragment.distribution_type {
1888 DistributionType::Single => {
1889 let actor = Itertools::exactly_one(fragment.actors.values()).unwrap();
1890 WorkerSlotMapping::new_single(WorkerSlotId::new(actor.worker_id as _, 0))
1891 }
1892 DistributionType::Hash => {
1893 let actor_bitmaps: HashMap<_, _> = fragment
1894 .actors
1895 .iter()
1896 .map(|(actor_id, actor_info)| {
1897 let vnode_bitmap = actor_info
1898 .vnode_bitmap
1899 .as_ref()
1900 .cloned()
1901 .expect("actor bitmap shouldn't be none in hash fragment");
1902
1903 (*actor_id as hash::ActorId, vnode_bitmap)
1904 })
1905 .collect();
1906
1907 let actor_mapping = ActorMapping::from_bitmaps(&actor_bitmaps);
1908
1909 let actor_locations = fragment
1910 .actors
1911 .iter()
1912 .map(|(actor_id, actor_info)| (*actor_id as hash::ActorId, actor_info.worker_id))
1913 .collect();
1914
1915 actor_mapping.to_worker_slot(&actor_locations)
1916 }
1917 };
1918
1919 PbFragmentWorkerSlotMapping {
1920 fragment_id: fragment.fragment_id,
1921 mapping: Some(fragment_worker_slot_mapping.to_protobuf()),
1922 }
1923}
1924
1925pub async fn get_fragments_for_jobs<C>(
1930 db: &C,
1931 streaming_jobs: Vec<JobId>,
1932) -> MetaResult<(
1933 HashMap<SourceId, BTreeSet<FragmentId>>,
1934 HashSet<FragmentId>,
1935 HashSet<FragmentId>,
1936)>
1937where
1938 C: ConnectionTrait,
1939{
1940 if streaming_jobs.is_empty() {
1941 return Ok((HashMap::default(), HashSet::default(), HashSet::default()));
1942 }
1943
1944 let fragments: Vec<(FragmentId, i32, StreamNode)> = Fragment::find()
1945 .select_only()
1946 .columns([
1947 fragment::Column::FragmentId,
1948 fragment::Column::FragmentTypeMask,
1949 fragment::Column::StreamNode,
1950 ])
1951 .filter(fragment::Column::JobId.is_in(streaming_jobs))
1952 .into_tuple()
1953 .all(db)
1954 .await?;
1955
1956 let fragment_ids: HashSet<_> = fragments
1957 .iter()
1958 .map(|(fragment_id, _, _)| *fragment_id)
1959 .collect();
1960
1961 let mut source_fragment_ids: HashMap<SourceId, BTreeSet<FragmentId>> = HashMap::new();
1962 let mut sink_fragment_ids: HashSet<FragmentId> = HashSet::new();
1963 for (fragment_id, mask, stream_node) in fragments {
1964 if FragmentTypeMask::from(mask).contains(FragmentTypeFlag::Source)
1965 && let Some(source_id) = stream_node.to_protobuf().find_stream_source()
1966 {
1967 source_fragment_ids
1968 .entry(source_id)
1969 .or_default()
1970 .insert(fragment_id);
1971 }
1972 if FragmentTypeMask::from(mask).contains(FragmentTypeFlag::Sink) {
1973 sink_fragment_ids.insert(fragment_id);
1974 }
1975 }
1976
1977 Ok((source_fragment_ids, sink_fragment_ids, fragment_ids))
1978}
1979
1980pub(crate) fn build_object_group_for_delete(
1985 partial_objects: Vec<PartialObject>,
1986) -> NotificationInfo {
1987 let mut objects = vec![];
1988 for obj in partial_objects {
1989 match obj.obj_type {
1990 ObjectType::Database => objects.push(PbObject {
1991 object_info: Some(PbObjectInfo::Database(PbDatabase {
1992 id: obj.oid.as_database_id(),
1993 ..Default::default()
1994 })),
1995 }),
1996 ObjectType::Schema => objects.push(PbObject {
1997 object_info: Some(PbObjectInfo::Schema(PbSchema {
1998 id: obj.oid.as_schema_id(),
1999 database_id: obj.database_id.unwrap(),
2000 ..Default::default()
2001 })),
2002 }),
2003 ObjectType::Table => objects.push(PbObject {
2004 object_info: Some(PbObjectInfo::Table(PbTable {
2005 id: obj.oid.as_table_id(),
2006 schema_id: obj.schema_id.unwrap(),
2007 database_id: obj.database_id.unwrap(),
2008 ..Default::default()
2009 })),
2010 }),
2011 ObjectType::Source => objects.push(PbObject {
2012 object_info: Some(PbObjectInfo::Source(PbSource {
2013 id: obj.oid.as_source_id(),
2014 schema_id: obj.schema_id.unwrap(),
2015 database_id: obj.database_id.unwrap(),
2016 ..Default::default()
2017 })),
2018 }),
2019 ObjectType::Sink => objects.push(PbObject {
2020 object_info: Some(PbObjectInfo::Sink(PbSink {
2021 id: obj.oid.as_sink_id(),
2022 schema_id: obj.schema_id.unwrap(),
2023 database_id: obj.database_id.unwrap(),
2024 ..Default::default()
2025 })),
2026 }),
2027 ObjectType::Subscription => objects.push(PbObject {
2028 object_info: Some(PbObjectInfo::Subscription(PbSubscription {
2029 id: obj.oid.as_subscription_id(),
2030 schema_id: obj.schema_id.unwrap(),
2031 database_id: obj.database_id.unwrap(),
2032 ..Default::default()
2033 })),
2034 }),
2035 ObjectType::View => objects.push(PbObject {
2036 object_info: Some(PbObjectInfo::View(PbView {
2037 id: obj.oid.as_view_id(),
2038 schema_id: obj.schema_id.unwrap(),
2039 database_id: obj.database_id.unwrap(),
2040 ..Default::default()
2041 })),
2042 }),
2043 ObjectType::Index => {
2044 objects.push(PbObject {
2045 object_info: Some(PbObjectInfo::Index(PbIndex {
2046 id: obj.oid.as_index_id(),
2047 schema_id: obj.schema_id.unwrap(),
2048 database_id: obj.database_id.unwrap(),
2049 ..Default::default()
2050 })),
2051 });
2052 objects.push(PbObject {
2053 object_info: Some(PbObjectInfo::Table(PbTable {
2054 id: obj.oid.as_table_id(),
2055 schema_id: obj.schema_id.unwrap(),
2056 database_id: obj.database_id.unwrap(),
2057 ..Default::default()
2058 })),
2059 });
2060 }
2061 ObjectType::Function => objects.push(PbObject {
2062 object_info: Some(PbObjectInfo::Function(PbFunction {
2063 id: obj.oid.as_function_id(),
2064 schema_id: obj.schema_id.unwrap(),
2065 database_id: obj.database_id.unwrap(),
2066 ..Default::default()
2067 })),
2068 }),
2069 ObjectType::Connection => objects.push(PbObject {
2070 object_info: Some(PbObjectInfo::Connection(PbConnection {
2071 id: obj.oid.as_connection_id(),
2072 schema_id: obj.schema_id.unwrap(),
2073 database_id: obj.database_id.unwrap(),
2074 ..Default::default()
2075 })),
2076 }),
2077 ObjectType::Secret => objects.push(PbObject {
2078 object_info: Some(PbObjectInfo::Secret(PbSecret {
2079 id: obj.oid.as_secret_id(),
2080 schema_id: obj.schema_id.unwrap(),
2081 database_id: obj.database_id.unwrap(),
2082 ..Default::default()
2083 })),
2084 }),
2085 }
2086 }
2087 NotificationInfo::ObjectGroup(PbObjectGroup {
2088 objects,
2089 dependencies: vec![],
2090 })
2091}
2092
2093pub fn extract_external_table_name_from_definition(table_definition: &str) -> Option<String> {
2094 let [mut definition]: [_; 1] = Parser::parse_sql(table_definition)
2095 .context("unable to parse table definition")
2096 .inspect_err(|e| {
2097 tracing::error!(
2098 target: "auto_schema_change",
2099 error = %e.as_report(),
2100 "failed to parse table definition")
2101 })
2102 .ok()?
2103 .try_into()
2104 .unwrap();
2105 if let SqlStatement::CreateTable { cdc_table_info, .. } = &mut definition {
2106 cdc_table_info
2107 .clone()
2108 .map(|cdc_table_info| cdc_table_info.external_table_name)
2109 } else {
2110 None
2111 }
2112}
2113
2114pub fn extract_cdc_excluded_columns_from_definition(table_definition: &str) -> HashSet<String> {
2117 let Ok(SqlStatement::CreateTable { with_options, .. }) =
2118 Parser::parse_exactly_one(table_definition)
2119 else {
2120 return HashSet::new();
2121 };
2122 with_options
2123 .iter()
2124 .find(|o| o.name.real_value() == CDC_COLUMN_EXCLUDE_LIST)
2125 .and_then(|o| match &o.value {
2126 SqlOptionValue::Value(Value::SingleQuotedString(s))
2127 | SqlOptionValue::Value(Value::CstyleEscapedString(CstyleEscapedString {
2128 value: s,
2129 ..
2130 })) => Some(parse_cdc_column_exclude_list(s)),
2131 _ => None,
2132 })
2133 .unwrap_or_default()
2134}
2135
2136pub async fn rename_relation(
2139 txn: &DatabaseTransaction,
2140 object_type: ObjectType,
2141 object_id: ObjectId,
2142 object_name: &str,
2143) -> MetaResult<(Vec<PbObject>, String)> {
2144 use sea_orm::ActiveModelTrait;
2145
2146 use crate::controller::rename::alter_relation_rename;
2147
2148 let mut to_update_relations = vec![];
2149 macro_rules! rename_relation {
2151 ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
2152 let (mut relation, obj) = $entity::find_by_id($object_id)
2153 .find_also_related(Object)
2154 .one(txn)
2155 .await?
2156 .unwrap();
2157 let obj = obj.unwrap();
2158 let old_name = relation.name.clone();
2159 relation.name = object_name.into();
2160 if obj.obj_type != ObjectType::View {
2161 relation.definition = alter_relation_rename(&relation.definition, object_name)
2162 .with_context(|| format!("failed to rename {old_name}"))?;
2163 }
2164 let active_model = $table::ActiveModel {
2165 $identity: Set(relation.$identity),
2166 name: Set(object_name.into()),
2167 definition: Set(relation.definition.clone()),
2168 ..Default::default()
2169 };
2170 active_model.update(txn).await?;
2171 let streaming_job = streaming_job::Entity::find_by_id($object_id.as_raw_id())
2172 .one(txn)
2173 .await?;
2174 to_update_relations.push(PbObject {
2175 object_info: Some(PbObjectInfo::$entity(
2176 ObjectModel(relation, obj, streaming_job).into(),
2177 )),
2178 });
2179 old_name
2180 }};
2181 }
2182 let old_name = match object_type {
2184 ObjectType::Table => {
2185 let associated_source_id: Option<SourceId> = Source::find()
2186 .select_only()
2187 .column(source::Column::SourceId)
2188 .filter(source::Column::OptionalAssociatedTableId.eq(object_id))
2189 .into_tuple()
2190 .one(txn)
2191 .await?;
2192 if let Some(source_id) = associated_source_id {
2193 rename_relation!(Source, source, source_id, source_id);
2194 }
2195 rename_relation!(Table, table, table_id, object_id.as_table_id())
2196 }
2197 ObjectType::Source => {
2198 rename_relation!(Source, source, source_id, object_id.as_source_id())
2199 }
2200 ObjectType::Sink => rename_relation!(Sink, sink, sink_id, object_id.as_sink_id()),
2201 ObjectType::Subscription => {
2202 rename_relation!(
2203 Subscription,
2204 subscription,
2205 subscription_id,
2206 object_id.as_subscription_id()
2207 )
2208 }
2209 ObjectType::View => rename_relation!(View, view, view_id, object_id.as_view_id()),
2210 ObjectType::Index => {
2211 let (mut index, obj) = Index::find_by_id(object_id.as_index_id())
2212 .find_also_related(Object)
2213 .one(txn)
2214 .await?
2215 .unwrap();
2216 let streaming_job = streaming_job::Entity::find_by_id(index.index_id.as_job_id())
2217 .one(txn)
2218 .await?;
2219 index.name = object_name.into();
2220 let index_table_id = index.index_table_id;
2221 let old_name = rename_relation!(Table, table, table_id, index_table_id);
2222
2223 let active_model = index::ActiveModel {
2225 index_id: sea_orm::ActiveValue::Set(index.index_id),
2226 name: sea_orm::ActiveValue::Set(object_name.into()),
2227 ..Default::default()
2228 };
2229 active_model.update(txn).await?;
2230 to_update_relations.push(PbObject {
2231 object_info: Some(PbObjectInfo::Index(
2232 ObjectModel(index, obj.unwrap(), streaming_job).into(),
2233 )),
2234 });
2235 old_name
2236 }
2237 _ => unreachable!("only relation name can be altered."),
2238 };
2239
2240 Ok((to_update_relations, old_name))
2241}
2242
2243pub async fn get_database_resource_group<C>(txn: &C, database_id: DatabaseId) -> MetaResult<String>
2244where
2245 C: ConnectionTrait,
2246{
2247 let database_resource_group: Option<String> = Database::find_by_id(database_id)
2248 .select_only()
2249 .column(database::Column::ResourceGroup)
2250 .into_tuple()
2251 .one(txn)
2252 .await?
2253 .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))?;
2254
2255 Ok(database_resource_group.unwrap_or_else(|| DEFAULT_RESOURCE_GROUP.to_owned()))
2256}
2257
2258pub async fn get_existing_job_resource_group<C>(
2259 txn: &C,
2260 streaming_job_id: JobId,
2261) -> MetaResult<String>
2262where
2263 C: ConnectionTrait,
2264{
2265 let (job_specific_resource_group, database_resource_group): (Option<String>, Option<String>) =
2266 StreamingJob::find_by_id(streaming_job_id)
2267 .select_only()
2268 .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
2269 .join(JoinType::InnerJoin, object::Relation::Database2.def())
2270 .column(streaming_job::Column::SpecificResourceGroup)
2271 .column(database::Column::ResourceGroup)
2272 .into_tuple()
2273 .one(txn)
2274 .await?
2275 .ok_or_else(|| MetaError::catalog_id_not_found("streaming job", streaming_job_id))?;
2276
2277 Ok(job_specific_resource_group.unwrap_or_else(|| {
2278 database_resource_group.unwrap_or_else(|| DEFAULT_RESOURCE_GROUP.to_owned())
2279 }))
2280}
2281
2282pub fn filter_workers_by_resource_group(
2283 workers: &HashMap<WorkerId, WorkerNode>,
2284 resource_group: &str,
2285) -> BTreeSet<WorkerId> {
2286 workers
2287 .iter()
2288 .filter(|&(_, worker)| {
2289 worker
2290 .resource_group()
2291 .map(|node_label| node_label.as_str() == resource_group)
2292 .unwrap_or(false)
2293 })
2294 .map(|(id, _)| *id)
2295 .collect()
2296}
2297
2298pub async fn rename_relation_refer(
2301 txn: &DatabaseTransaction,
2302 object_type: ObjectType,
2303 object_id: ObjectId,
2304 object_name: &str,
2305 old_name: &str,
2306) -> MetaResult<Vec<PbObject>> {
2307 use sea_orm::ActiveModelTrait;
2308
2309 use crate::controller::rename::alter_relation_rename_refs;
2310
2311 let mut to_update_relations = vec![];
2312 macro_rules! rename_relation_ref {
2313 ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
2314 let (mut relation, obj) = $entity::find_by_id($object_id)
2315 .find_also_related(Object)
2316 .one(txn)
2317 .await?
2318 .unwrap();
2319 relation.definition =
2320 alter_relation_rename_refs(&relation.definition, old_name, object_name)
2321 .with_context(|| {
2322 format!("failed to update the definition of {}", relation.name)
2323 })?;
2324 let active_model = $table::ActiveModel {
2325 $identity: Set(relation.$identity),
2326 definition: Set(relation.definition.clone()),
2327 ..Default::default()
2328 };
2329 active_model.update(txn).await?;
2330 let streaming_job = streaming_job::Entity::find_by_id($object_id.as_raw_id())
2331 .one(txn)
2332 .await?;
2333 to_update_relations.push(PbObject {
2334 object_info: Some(PbObjectInfo::$entity(
2335 ObjectModel(relation, obj.unwrap(), streaming_job).into(),
2336 )),
2337 });
2338 }};
2339 }
2340 let objs = get_referring_objects(object_id, object_type, txn).await?;
2341
2342 for obj in objs {
2343 match obj.obj_type {
2344 ObjectType::Table => {
2345 rename_relation_ref!(Table, table, table_id, obj.oid.as_table_id())
2346 }
2347 ObjectType::Sink => {
2348 rename_relation_ref!(Sink, sink, sink_id, obj.oid.as_sink_id())
2349 }
2350 ObjectType::Source => {
2352 rename_relation_ref!(Source, source, source_id, obj.oid.as_source_id())
2353 }
2354 ObjectType::Subscription => {
2355 rename_relation_ref!(
2356 Subscription,
2357 subscription,
2358 subscription_id,
2359 obj.oid.as_subscription_id()
2360 )
2361 }
2362 ObjectType::View => rename_relation_ref!(View, view, view_id, obj.oid.as_view_id()),
2363 ObjectType::Index => {
2364 let index_table_id: Option<TableId> = Index::find_by_id(obj.oid.as_index_id())
2365 .select_only()
2366 .column(index::Column::IndexTableId)
2367 .into_tuple()
2368 .one(txn)
2369 .await?;
2370 rename_relation_ref!(Table, table, table_id, index_table_id.unwrap());
2371 }
2372 _ => {
2373 bail!(
2374 "only the table, source, sink, subscription, view and index will depend on other objects."
2375 )
2376 }
2377 }
2378 }
2379
2380 Ok(to_update_relations)
2381}
2382
2383pub async fn validate_subscription_deletion<C>(
2387 txn: &C,
2388 subscription_id: SubscriptionId,
2389) -> MetaResult<()>
2390where
2391 C: ConnectionTrait,
2392{
2393 let upstream_table_id: ObjectId = Subscription::find_by_id(subscription_id)
2394 .select_only()
2395 .column(subscription::Column::DependentTableId)
2396 .into_tuple()
2397 .one(txn)
2398 .await?
2399 .ok_or_else(|| MetaError::catalog_id_not_found("subscription", subscription_id))?;
2400
2401 let cnt = Subscription::find()
2402 .filter(subscription::Column::DependentTableId.eq(upstream_table_id))
2403 .count(txn)
2404 .await?;
2405 if cnt > 1 {
2406 return Ok(());
2409 }
2410
2411 let obj_alias = Alias::new("o1");
2413 let used_by_alias = Alias::new("o2");
2414 let count = ObjectDependency::find()
2415 .join_as(
2416 JoinType::InnerJoin,
2417 object_dependency::Relation::Object2.def(),
2418 obj_alias.clone(),
2419 )
2420 .join_as(
2421 JoinType::InnerJoin,
2422 object_dependency::Relation::Object1.def(),
2423 used_by_alias.clone(),
2424 )
2425 .filter(
2426 object_dependency::Column::Oid
2427 .eq(upstream_table_id)
2428 .and(object_dependency::Column::UsedBy.ne(subscription_id))
2429 .and(
2430 Expr::col((obj_alias, object::Column::DatabaseId))
2431 .ne(Expr::col((used_by_alias, object::Column::DatabaseId))),
2432 ),
2433 )
2434 .count(txn)
2435 .await?;
2436
2437 if count != 0 {
2438 return Err(MetaError::permission_denied(format!(
2439 "Referenced by {} cross-db objects.",
2440 count
2441 )));
2442 }
2443
2444 Ok(())
2445}
2446
2447pub async fn fetch_target_fragments<C>(
2448 txn: &C,
2449 src_fragment_id: impl IntoIterator<Item = FragmentId>,
2450) -> MetaResult<HashMap<FragmentId, Vec<FragmentId>>>
2451where
2452 C: ConnectionTrait,
2453{
2454 let source_target_fragments: Vec<(FragmentId, FragmentId)> = FragmentRelation::find()
2455 .select_only()
2456 .columns([
2457 fragment_relation::Column::SourceFragmentId,
2458 fragment_relation::Column::TargetFragmentId,
2459 ])
2460 .filter(fragment_relation::Column::SourceFragmentId.is_in(src_fragment_id))
2461 .into_tuple()
2462 .all(txn)
2463 .await?;
2464
2465 let source_target_fragments = source_target_fragments.into_iter().into_group_map();
2466
2467 Ok(source_target_fragments)
2468}
2469
2470pub async fn get_sink_fragment_by_ids<C>(
2471 txn: &C,
2472 sink_ids: Vec<SinkId>,
2473) -> MetaResult<HashMap<SinkId, FragmentId>>
2474where
2475 C: ConnectionTrait,
2476{
2477 let sink_num = sink_ids.len();
2478 let sink_fragment_ids: Vec<(SinkId, FragmentId)> = Fragment::find()
2479 .select_only()
2480 .columns([fragment::Column::JobId, fragment::Column::FragmentId])
2481 .filter(
2482 fragment::Column::JobId
2483 .is_in(sink_ids)
2484 .and(FragmentTypeMask::intersects(FragmentTypeFlag::Sink)),
2485 )
2486 .into_tuple()
2487 .all(txn)
2488 .await?;
2489
2490 if sink_fragment_ids.len() != sink_num {
2491 return Err(anyhow::anyhow!(
2492 "expected exactly one sink fragment for each sink, but got {} fragments for {} sinks",
2493 sink_fragment_ids.len(),
2494 sink_num
2495 )
2496 .into());
2497 }
2498
2499 Ok(sink_fragment_ids.into_iter().collect())
2500}
2501
2502pub async fn has_table_been_migrated<C>(txn: &C, table_id: TableId) -> MetaResult<bool>
2503where
2504 C: ConnectionTrait,
2505{
2506 let mview_fragment: Vec<i32> = Fragment::find()
2507 .select_only()
2508 .column(fragment::Column::FragmentTypeMask)
2509 .filter(
2510 fragment::Column::JobId
2511 .eq(table_id)
2512 .and(FragmentTypeMask::intersects(FragmentTypeFlag::Mview)),
2513 )
2514 .into_tuple()
2515 .all(txn)
2516 .await?;
2517
2518 let mview_fragment_len = mview_fragment.len();
2519 if mview_fragment_len != 1 {
2520 bail!(
2521 "expected exactly one mview fragment for table {}, found {}",
2522 table_id,
2523 mview_fragment_len
2524 );
2525 }
2526
2527 let mview_fragment = mview_fragment.into_iter().next().unwrap();
2528 let migrated =
2529 FragmentTypeMask::from(mview_fragment).contains(FragmentTypeFlag::UpstreamSinkUnion);
2530
2531 Ok(migrated)
2532}
2533
2534pub async fn check_if_belongs_to_iceberg_table<C>(txn: &C, job_id: JobId) -> MetaResult<bool>
2535where
2536 C: ConnectionTrait,
2537{
2538 if let Some(engine) = Table::find_by_id(job_id.as_mv_table_id())
2539 .select_only()
2540 .column(table::Column::Engine)
2541 .into_tuple::<table::Engine>()
2542 .one(txn)
2543 .await?
2544 && engine == table::Engine::Iceberg
2545 {
2546 return Ok(true);
2547 }
2548 if let Some(parent_oid) = Object::find_by_id(job_id)
2549 .select_only()
2550 .column(object::Column::BelongToOid)
2551 .into_tuple::<Option<ObjectId>>()
2552 .one(txn)
2553 .await?
2554 .flatten()
2555 && Table::find_by_id(parent_oid.as_table_id())
2556 .filter(table::Column::Engine.eq(table::Engine::Iceberg))
2557 .one(txn)
2558 .await?
2559 .is_some()
2560 {
2561 return Ok(true);
2562 }
2563 Ok(false)
2564}
2565
2566pub async fn find_dirty_iceberg_table_jobs<C>(
2567 txn: &C,
2568 database_id: Option<DatabaseId>,
2569) -> MetaResult<Vec<PartialObject>>
2570where
2571 C: ConnectionTrait,
2572{
2573 let mut filter_condition = streaming_job::Column::JobStatus
2574 .ne(JobStatus::Created)
2575 .and(object::Column::ObjType.is_in([ObjectType::Table, ObjectType::Sink]))
2576 .and(streaming_job::Column::CreateType.eq(CreateType::Background));
2577 if let Some(database_id) = database_id {
2578 filter_condition = filter_condition.and(object::Column::DatabaseId.eq(database_id));
2579 }
2580 let creating_table_sink_jobs: Vec<PartialObject> = StreamingJob::find()
2581 .select_only()
2582 .columns([
2583 object::Column::Oid,
2584 object::Column::ObjType,
2585 object::Column::SchemaId,
2586 object::Column::DatabaseId,
2587 ])
2588 .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
2589 .filter(filter_condition)
2590 .into_partial_model()
2591 .all(txn)
2592 .await?;
2593
2594 let mut dirty_iceberg_table_jobs = vec![];
2595 for job in creating_table_sink_jobs {
2596 if check_if_belongs_to_iceberg_table(txn, job.oid.as_job_id()).await? {
2597 tracing::info!("Found dirty iceberg job with id: {}", job.oid);
2598 dirty_iceberg_table_jobs.push(job);
2599 }
2600 }
2601
2602 Ok(dirty_iceberg_table_jobs)
2603}
2604
2605pub fn build_select_node_list(
2606 from: &[ColumnCatalog],
2607 to: &[ColumnCatalog],
2608) -> MetaResult<Vec<PbExprNode>> {
2609 let mut exprs = Vec::with_capacity(to.len());
2610 let idx_by_col_id = from
2611 .iter()
2612 .enumerate()
2613 .map(|(idx, col)| (col.column_desc.as_ref().unwrap().column_id, idx))
2614 .collect::<HashMap<_, _>>();
2615
2616 for to_col in to {
2617 let to_col = to_col.column_desc.as_ref().unwrap();
2618 let to_col_type_ref = to_col.column_type.as_ref().unwrap();
2619 let to_col_type = DataType::from(to_col_type_ref);
2620 if let Some(from_idx) = idx_by_col_id.get(&to_col.column_id) {
2621 let from_col_type = DataType::from(
2622 from[*from_idx]
2623 .column_desc
2624 .as_ref()
2625 .unwrap()
2626 .column_type
2627 .as_ref()
2628 .unwrap(),
2629 );
2630 if !to_col_type.equals_datatype(&from_col_type) {
2631 return Err(anyhow!(
2632 "Column type mismatch: {:?} != {:?}",
2633 from_col_type,
2634 to_col_type
2635 )
2636 .into());
2637 }
2638 exprs.push(PbExprNode {
2639 function_type: expr_node::Type::Unspecified.into(),
2640 return_type: Some(to_col_type_ref.clone()),
2641 rex_node: Some(expr_node::RexNode::InputRef(*from_idx as _)),
2642 });
2643 } else {
2644 let to_default_node =
2645 if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
2646 expr,
2647 ..
2648 })) = &to_col.generated_or_default_column
2649 {
2650 expr.clone().unwrap()
2651 } else {
2652 let null = Datum::None.to_protobuf();
2653 PbExprNode {
2654 function_type: expr_node::Type::Unspecified.into(),
2655 return_type: Some(to_col_type_ref.clone()),
2656 rex_node: Some(expr_node::RexNode::Constant(null)),
2657 }
2658 };
2659 exprs.push(to_default_node);
2660 }
2661 }
2662
2663 Ok(exprs)
2664}
2665
2666#[derive(Clone, Debug, Default)]
2667pub struct StreamingJobExtraInfo {
2668 pub timezone: Option<String>,
2669 pub config_override: Arc<str>,
2670 pub job_definition: String,
2671 pub backfill_orders: Option<BackfillOrders>,
2672 pub refresh_interval_sec: Option<u64>,
2673}
2674
2675impl StreamingJobExtraInfo {
2676 pub fn stream_context(&self) -> StreamContext {
2677 StreamContext {
2678 timezone: self.timezone.clone(),
2679 config_override: self.config_override.clone(),
2680 }
2681 }
2682}
2683
2684type StreamingJobExtraInfoRow = (
2686 JobId,
2687 Option<String>,
2688 Option<String>,
2689 Option<BackfillOrders>,
2690 Option<i64>,
2691);
2692
2693pub async fn get_streaming_job_extra_info<C>(
2694 txn: &C,
2695 job_ids: Vec<JobId>,
2696) -> MetaResult<HashMap<JobId, StreamingJobExtraInfo>>
2697where
2698 C: ConnectionTrait,
2699{
2700 let pairs: Vec<StreamingJobExtraInfoRow> = StreamingJob::find()
2701 .select_only()
2702 .columns([
2703 streaming_job::Column::JobId,
2704 streaming_job::Column::Timezone,
2705 streaming_job::Column::ConfigOverride,
2706 streaming_job::Column::BackfillOrders,
2707 streaming_job::Column::RefreshIntervalSec,
2708 ])
2709 .filter(streaming_job::Column::JobId.is_in(job_ids.clone()))
2710 .into_tuple()
2711 .all(txn)
2712 .await?;
2713
2714 let job_ids = job_ids.into_iter().collect();
2715
2716 let mut definitions = resolve_streaming_job_definition(txn, &job_ids).await?;
2717
2718 let result = pairs
2719 .into_iter()
2720 .map(
2721 |(job_id, timezone, config_override, backfill_orders, refresh_interval_sec)| {
2722 let job_definition = definitions.remove(&job_id).unwrap_or_default();
2723 (
2724 job_id,
2725 StreamingJobExtraInfo {
2726 timezone,
2727 config_override: config_override.unwrap_or_default().into(),
2728 job_definition,
2729 backfill_orders,
2730 refresh_interval_sec: refresh_interval_sec.map(|s| s as u64),
2731 },
2732 )
2733 },
2734 )
2735 .collect();
2736
2737 Ok(result)
2738}
2739
2740#[cfg(test)]
2741mod tests {
2742 use super::*;
2743
2744 #[test]
2745 fn test_extract_cdc_table_name() {
2746 let ddl1 = "CREATE TABLE t1 () FROM pg_source TABLE 'public.t1'";
2747 let ddl2 = "CREATE TABLE t2 (v1 int) FROM pg_source TABLE 'mydb.t2'";
2748 assert_eq!(
2749 extract_external_table_name_from_definition(ddl1),
2750 Some("public.t1".into())
2751 );
2752 assert_eq!(
2753 extract_external_table_name_from_definition(ddl2),
2754 Some("mydb.t2".into())
2755 );
2756 }
2757}