1use std::collections::HashSet;
16use std::sync::Arc;
17use std::time::Instant;
18
19use itertools::Itertools;
20use pgwire::pg_field_descriptor::PgFieldDescriptor;
21use pgwire::pg_response::{PgResponse, StatementType};
22use pgwire::types::Format;
23use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
24use risingwave_common::bail_not_implemented;
25use risingwave_common::catalog::{FunctionId, Schema, SecretId};
26use risingwave_common::id::ObjectId;
27use risingwave_common::session_config::QueryMode;
28use risingwave_common::types::{DataType, Datum};
29use risingwave_sqlparser::ast::{SetExpr, Statement};
30
31use super::extended_handle::{PortalResult, PrepareStatement, PreparedResult};
32use super::{PgResponseStream, RwPgResponse, create_mv, declare_cursor};
33use crate::binder::{Binder, BoundCreateView, BoundStatement};
34#[cfg(feature = "datafusion")]
35use crate::datafusion::DfBatchQueryPlanResult;
36use crate::error::{ErrorCode, Result, RwError};
37use crate::handler::HandlerArgs;
38use crate::handler::flush::do_flush;
39use crate::handler::util::{DataChunkToRowSetAdapter, to_pg_field};
40use crate::optimizer::plan_node::{BatchPlanRef, Explain};
41use crate::optimizer::{
42 BatchPlanRoot, ExecutionModeDecider, OptimizerContext, OptimizerContextRef, SysTableVisitor,
43};
44use crate::planner::Planner;
45use crate::scheduler::plan_fragmenter::Query;
46use crate::scheduler::{
47 BatchPlanFragmenter, DistributedQueryStream, ExecutionContext, ExecutionContextRef,
48 LocalQueryExecution, LocalQueryStream, ReadSnapshot,
49};
50use crate::session::SessionImpl;
51
52pub enum BatchPlanChoice {
54 Rw(RwBatchQueryPlanResult),
55 #[cfg(feature = "datafusion")]
56 Df(DfBatchQueryPlanResult),
57}
58
59impl BatchPlanChoice {
60 pub fn unwrap_rw(self) -> Result<RwBatchQueryPlanResult> {
61 match self {
62 BatchPlanChoice::Rw(result) => Ok(result),
63 #[cfg(feature = "datafusion")]
64 BatchPlanChoice::Df { .. } => {
65 risingwave_common::bail!(
66 "Expected RisingWave plan in BatchPlanChoice, but got DataFusion plan"
67 )
68 }
69 }
70 }
71}
72
73pub async fn handle_query(
74 handler_args: HandlerArgs,
75 stmt: Statement,
76 formats: Vec<Format>,
77) -> Result<RwPgResponse> {
78 let session = handler_args.session.clone();
79 let context = OptimizerContext::from_handler_args(handler_args)?;
80
81 #[cfg(feature = "datafusion")]
82 {
83 use futures::FutureExt;
87
88 use crate::datafusion::execute_datafusion_plan;
89
90 let future = match gen_batch_plan_by_statement(&session, context.into(), stmt)? {
91 BatchPlanChoice::Rw(plan_result) => {
92 let plan_fragmenter_result = risingwave_expr::expr_context::TIME_ZONE::sync_scope(
93 session.config().timezone(),
94 || gen_batch_plan_fragmenter(&session, plan_result),
95 )?;
96 execute_risingwave_plan(session, plan_fragmenter_result, formats).left_future()
97 }
98 BatchPlanChoice::Df(plan_result) => {
99 execute_datafusion_plan(session, plan_result, formats).right_future()
100 }
101 };
102 future.await
103 }
104
105 #[cfg(not(feature = "datafusion"))]
106 {
107 let future = match gen_batch_plan_by_statement(&session, context.into(), stmt)? {
108 BatchPlanChoice::Rw(plan_result) => {
109 let plan_fragmenter_result = risingwave_expr::expr_context::TIME_ZONE::sync_scope(
110 session.config().timezone(),
111 || gen_batch_plan_fragmenter(&session, plan_result),
112 )?;
113 execute_risingwave_plan(session, plan_fragmenter_result, formats)
114 }
115 };
116 future.await
117 }
118}
119
120fn handle_parse_inner(binder: Binder, statement: Statement) -> Result<PrepareStatement> {
121 let bound_result = gen_bound(binder, statement.clone())?;
122
123 Ok(PrepareStatement::Prepared(PreparedResult {
124 statement,
125 bound_result,
126 }))
127}
128
129pub fn handle_parse_for_batch(
130 handler_args: HandlerArgs,
131 statement: Statement,
132 specified_param_types: Vec<Option<DataType>>,
133) -> Result<PrepareStatement> {
134 let binder = Binder::new_for_batch(&handler_args.session)
135 .with_specified_params_types(specified_param_types);
136 handle_parse_inner(binder, statement)
137}
138
139pub fn handle_parse_for_stream(
140 handler_args: HandlerArgs,
141 statement: Statement,
142 specified_param_types: Vec<Option<DataType>>,
143) -> Result<PrepareStatement> {
144 let binder = Binder::new_for_stream(&handler_args.session)
145 .with_specified_params_types(specified_param_types);
146 handle_parse_inner(binder, statement)
147}
148
149pub async fn handle_execute(
151 handler_args: HandlerArgs,
152 portal: PortalResult,
153) -> Result<RwPgResponse> {
154 let PortalResult {
155 bound_result,
156 result_formats,
157 statement,
158 } = portal;
159 match statement {
160 Statement::Query(_)
161 | Statement::Insert { .. }
162 | Statement::Delete { .. }
163 | Statement::Update { .. } => {
164 let session = handler_args.session.clone();
166 let plan_fragmenter_result = {
167 let context = OptimizerContext::from_handler_args(handler_args)?;
168 let plan_result =
169 gen_batch_query_plan(&session, context.into(), bound_result)?.unwrap_rw()?;
170 risingwave_expr::expr_context::TIME_ZONE::sync_scope(
172 session.config().timezone(),
173 || gen_batch_plan_fragmenter(&session, plan_result),
174 )?
175 };
176 execute_risingwave_plan(session, plan_fragmenter_result, result_formats).await
177 }
178 Statement::CreateView { materialized, .. } if materialized => {
179 let BoundResult {
181 bound,
182 dependent_relations,
183 dependent_udfs,
184 dependent_secrets,
185 ..
186 } = bound_result;
187 let create_mv = if let BoundStatement::CreateView(create_mv) = bound {
188 create_mv
189 } else {
190 unreachable!("expect a BoundStatement::CreateView")
191 };
192 let BoundCreateView {
193 or_replace,
194 materialized: _,
195 if_not_exists,
196 name,
197 columns,
198 query,
199 emit_mode,
200 with_options,
201 } = *create_mv;
202 if or_replace {
203 bail_not_implemented!("CREATE OR REPLACE VIEW");
204 }
205
206 let handler_args = HandlerArgs {
208 session: handler_args.session.clone(),
209 sql: handler_args.sql.clone(),
210 normalized_sql: handler_args.normalized_sql.clone(),
211 with_options: crate::WithOptions::try_from(with_options.as_slice())?,
212 };
213
214 create_mv::handle_create_mv_bound(
215 handler_args,
216 if_not_exists,
217 name,
218 *query,
219 dependent_relations,
220 dependent_udfs,
221 dependent_secrets,
222 columns,
223 emit_mode,
224 )
225 .await
226 }
227 Statement::DeclareCursor { stmt } => match stmt.declare_cursor {
228 risingwave_sqlparser::ast::DeclareCursor::Query(_) => {
229 let session = handler_args.session.clone();
230 let plan_fragmenter_result = {
231 let context = OptimizerContext::from_handler_args(handler_args.clone())?;
232 let plan_result = gen_batch_query_plan(&session, context.into(), bound_result)?
233 .unwrap_rw()?;
234 gen_batch_plan_fragmenter(&session, plan_result)?
235 };
236 declare_cursor::handle_bound_declare_query_cursor(
237 handler_args,
238 stmt.cursor_name,
239 plan_fragmenter_result,
240 )
241 .await
242 }
243 risingwave_sqlparser::ast::DeclareCursor::Subscription(sub_name, rw_timestamp) => {
244 declare_cursor::handle_declare_subscription_cursor(
245 handler_args,
246 sub_name,
247 stmt.cursor_name,
248 rw_timestamp,
249 )
250 .await
251 }
252 },
253 _ => unreachable!(),
254 }
255}
256
257pub fn gen_batch_plan_by_statement(
258 session: &SessionImpl,
259 context: OptimizerContextRef,
260 stmt: Statement,
261) -> Result<BatchPlanChoice> {
262 let binder = Binder::new_for_batch(session);
263 let bound_result = gen_bound(binder, stmt)?;
264 gen_batch_query_plan(session, context, bound_result)
265}
266
267#[derive(Clone)]
268pub struct BoundResult {
269 pub(crate) stmt_type: StatementType,
270 pub(crate) must_dist: bool,
271 pub(crate) bound: BoundStatement,
272 pub(crate) param_types: Vec<DataType>,
273 pub(crate) parsed_params: Option<Vec<Datum>>,
274 pub(crate) dependent_relations: HashSet<ObjectId>,
275 pub(crate) dependent_udfs: HashSet<FunctionId>,
277 pub(crate) dependent_secrets: HashSet<SecretId>,
278}
279
280fn gen_bound(mut binder: Binder, stmt: Statement) -> Result<BoundResult> {
281 let stmt_type = StatementType::infer_from_statement(&stmt)
282 .map_err(|err| RwError::from(ErrorCode::InvalidInputSyntax(err)))?;
283 let must_dist = must_run_in_distributed_mode(&stmt)?;
284
285 let bound = binder.bind(stmt)?;
286
287 Ok(BoundResult {
288 stmt_type,
289 must_dist,
290 bound,
291 param_types: binder.export_param_types()?,
292 parsed_params: None,
293 dependent_relations: binder.included_relations().clone(),
294 dependent_udfs: binder.included_udfs().clone(),
295 dependent_secrets: binder.included_secrets().clone(),
296 })
297}
298
299pub struct RwBatchQueryPlanResult {
300 pub(crate) plan: BatchPlanRef,
301 pub(crate) query_mode: QueryMode,
302 pub(crate) schema: Schema,
303 pub(crate) stmt_type: StatementType,
304}
305
306fn gen_batch_query_plan(
307 session: &SessionImpl,
308 context: OptimizerContextRef,
309 bind_result: BoundResult,
310) -> Result<BatchPlanChoice> {
311 let BoundResult {
312 stmt_type,
313 must_dist,
314 bound,
315 ..
316 } = bind_result;
317
318 let mut planner = if matches!(bound, BoundStatement::Query(_)) {
319 Planner::new_for_batch_dql(context)
320 } else {
321 Planner::new_for_batch(context)
322 };
323
324 let logical = planner.plan(bound)?;
325 let schema = logical.schema();
326 let optimized_logical = logical.gen_optimized_logical_plan_for_batch()?;
327
328 #[cfg(feature = "datafusion")]
329 {
330 if session.config().enable_datafusion_engine() {
331 use thiserror_ext::AsReport;
332
333 use crate::datafusion::{GenDataFusionPlanError, try_gen_datafusion_plan};
334
335 match try_gen_datafusion_plan(&optimized_logical) {
336 Ok(plan) => {
337 return Ok(BatchPlanChoice::Df(DfBatchQueryPlanResult {
338 plan,
339 schema,
340 stmt_type,
341 }));
342 }
343 Err(GenDataFusionPlanError::MissingIcebergScan) => {}
344 Err(err) => {
345 tracing::warn!(
346 "Failed to generate DataFusion plan, fallback to RisingWave plan: {}",
347 err.as_report()
348 );
349 }
350 }
351 }
352 }
353
354 let batch_plan = optimized_logical.gen_batch_plan()?;
355
356 let must_local = must_run_in_local_mode(&batch_plan);
357
358 let query_mode = match (must_dist, must_local) {
359 (true, true) => {
360 return Err(ErrorCode::InternalError(
361 "the query is forced to both local and distributed mode by optimizer".to_owned(),
362 )
363 .into());
364 }
365 (true, false) => QueryMode::Distributed,
366 (false, true) => QueryMode::Local,
367 (false, false) => match session.config().query_mode() {
368 QueryMode::Auto => determine_query_mode(&batch_plan),
369 QueryMode::Local => QueryMode::Local,
370 QueryMode::Distributed => QueryMode::Distributed,
371 },
372 };
373
374 let physical = match query_mode {
375 QueryMode::Auto => unreachable!(),
376 QueryMode::Local => batch_plan.gen_batch_local_plan()?,
377 QueryMode::Distributed => batch_plan.gen_batch_distributed_plan()?,
378 };
379
380 let result = RwBatchQueryPlanResult {
381 plan: physical,
382 query_mode,
383 schema,
384 stmt_type,
385 };
386 Ok(BatchPlanChoice::Rw(result))
387}
388
389fn must_run_in_distributed_mode(stmt: &Statement) -> Result<bool> {
390 fn is_insert_using_select(stmt: &Statement) -> bool {
391 fn has_select_query(set_expr: &SetExpr) -> bool {
392 match set_expr {
393 SetExpr::Select(_) => true,
394 SetExpr::Query(query) => has_select_query(&query.body),
395 SetExpr::SetOperation { left, right, .. } => {
396 has_select_query(left) || has_select_query(right)
397 }
398 SetExpr::Values(_) => false,
399 }
400 }
401
402 matches!(
403 stmt,
404 Statement::Insert {source, ..} if has_select_query(&source.body)
405 )
406 }
407
408 let stmt_type = StatementType::infer_from_statement(stmt)
409 .map_err(|err| RwError::from(ErrorCode::InvalidInputSyntax(err)))?;
410
411 Ok(matches!(
412 stmt_type,
413 StatementType::UPDATE
414 | StatementType::DELETE
415 | StatementType::UPDATE_RETURNING
416 | StatementType::DELETE_RETURNING
417 ) | is_insert_using_select(stmt))
418}
419
420fn must_run_in_local_mode(batch_plan: &BatchPlanRoot) -> bool {
421 SysTableVisitor::has_sys_table(batch_plan)
422}
423
424fn determine_query_mode(batch_plan: &BatchPlanRoot) -> QueryMode {
425 if ExecutionModeDecider::run_in_local_mode(batch_plan) {
426 QueryMode::Local
427 } else {
428 QueryMode::Distributed
429 }
430}
431
432pub struct BatchPlanFragmenterResult {
433 pub(crate) plan_fragmenter: BatchPlanFragmenter,
434 pub(crate) query_mode: QueryMode,
435 pub(crate) schema: Schema,
436 pub(crate) stmt_type: StatementType,
437 pub(crate) snapshot_override: Option<ReadSnapshot>,
439}
440
441pub fn gen_batch_plan_fragmenter(
442 session: &SessionImpl,
443 plan_result: RwBatchQueryPlanResult,
444) -> Result<BatchPlanFragmenterResult> {
445 gen_batch_plan_fragmenter_with_snapshot(session, plan_result, None)
446}
447
448pub fn gen_batch_plan_fragmenter_with_snapshot(
449 session: &SessionImpl,
450 plan_result: RwBatchQueryPlanResult,
451 snapshot_override: Option<ReadSnapshot>,
452) -> Result<BatchPlanFragmenterResult> {
453 let RwBatchQueryPlanResult {
454 plan,
455 query_mode,
456 schema,
457 stmt_type,
458 ..
459 } = plan_result;
460
461 tracing::trace!(
462 "Generated query plan: {:?}, query_mode:{:?}",
463 plan.explain_to_string(),
464 query_mode
465 );
466 let worker_node_manager_reader = WorkerNodeSelector::new(
467 session.env().worker_node_manager_ref(),
468 snapshot_override
469 .as_ref()
470 .or(session.get_pinned_snapshot().as_ref())
471 .map_or_else(
472 || session.is_barrier_read(),
473 ReadSnapshot::support_barrier_read,
474 ),
475 );
476 let plan_fragmenter = BatchPlanFragmenter::new(
477 worker_node_manager_reader,
478 session.env().catalog_reader().clone(),
479 session.config().batch_parallelism().0,
480 plan,
481 )?;
482
483 Ok(BatchPlanFragmenterResult {
484 plan_fragmenter,
485 query_mode,
486 schema,
487 stmt_type,
488 snapshot_override,
489 })
490}
491
492pub async fn create_stream(
493 session: Arc<SessionImpl>,
494 plan_fragmenter_result: BatchPlanFragmenterResult,
495 formats: Vec<Format>,
496) -> Result<(PgResponseStream, Vec<PgFieldDescriptor>)> {
497 let BatchPlanFragmenterResult {
498 plan_fragmenter,
499 query_mode,
500 schema,
501 stmt_type,
502 snapshot_override,
503 } = plan_fragmenter_result;
504
505 let mut can_timeout_cancel = true;
506 match stmt_type {
508 StatementType::INSERT
509 | StatementType::INSERT_RETURNING
510 | StatementType::DELETE
511 | StatementType::DELETE_RETURNING
512 | StatementType::UPDATE
513 | StatementType::UPDATE_RETURNING => {
514 session.txn_write_guard()?;
515 can_timeout_cancel = false;
516 }
517 _ => {}
518 }
519
520 let query = plan_fragmenter.generate_complete_query().await?;
521 tracing::trace!("Generated query after plan fragmenter: {:?}", &query);
522
523 let pg_descs = schema
524 .fields()
525 .iter()
526 .map(to_pg_field)
527 .collect::<Vec<PgFieldDescriptor>>();
528 let column_types = schema.fields().iter().map(|f| f.data_type()).collect_vec();
529
530 let row_stream = match query_mode {
531 QueryMode::Auto => unreachable!(),
532 QueryMode::Local => PgResponseStream::LocalQuery(DataChunkToRowSetAdapter::new(
533 local_execute(
534 session.clone(),
535 query,
536 can_timeout_cancel,
537 snapshot_override,
538 )
539 .await?,
540 column_types,
541 formats,
542 session.clone(),
543 )),
544 QueryMode::Distributed => {
546 PgResponseStream::DistributedQuery(DataChunkToRowSetAdapter::new(
547 distribute_execute(
548 session.clone(),
549 query,
550 can_timeout_cancel,
551 snapshot_override,
552 )
553 .await?,
554 column_types,
555 formats,
556 session.clone(),
557 ))
558 }
559 };
560
561 Ok((row_stream, pg_descs))
562}
563
564async fn execute_risingwave_plan(
565 session: Arc<SessionImpl>,
566 plan_fragmenter_result: BatchPlanFragmenterResult,
567 formats: Vec<Format>,
568) -> Result<RwPgResponse> {
569 let first_field_format = formats.first().copied().unwrap_or(Format::Text);
571 let query_mode = plan_fragmenter_result.query_mode;
572 let stmt_type = plan_fragmenter_result.stmt_type;
573
574 let query_start_time = Instant::now();
575 let (row_stream, pg_descs) =
576 create_stream(session.clone(), plan_fragmenter_result, formats).await?;
577
578 let callback = async move {
581 if session.config().implicit_flush() && stmt_type.is_dml() {
583 do_flush(&session).await?;
584 }
585
586 match query_mode {
588 QueryMode::Auto => unreachable!(),
589 QueryMode::Local => {
590 session
591 .env()
592 .frontend_metrics
593 .latency_local_execution
594 .observe(query_start_time.elapsed().as_secs_f64());
595
596 session
597 .env()
598 .frontend_metrics
599 .query_counter_local_execution
600 .inc();
601 }
602 QueryMode::Distributed => {
603 session
604 .env()
605 .query_manager()
606 .query_metrics
607 .query_latency
608 .observe(query_start_time.elapsed().as_secs_f64());
609
610 session
611 .env()
612 .query_manager()
613 .query_metrics
614 .completed_query_counter
615 .inc();
616 }
617 }
618
619 Ok(())
620 };
621
622 Ok(PgResponse::builder(stmt_type)
623 .row_cnt_format_opt(Some(first_field_format))
624 .values(row_stream, pg_descs)
625 .callback(callback)
626 .into())
627}
628
629pub async fn distribute_execute(
630 session: Arc<SessionImpl>,
631 query: Query,
632 can_timeout_cancel: bool,
633 snapshot_override: Option<ReadSnapshot>,
634) -> Result<DistributedQueryStream> {
635 let timeout = if cfg!(madsim) {
636 None
637 } else if can_timeout_cancel {
638 Some(session.statement_timeout())
639 } else {
640 None
641 };
642 let execution_context: ExecutionContextRef =
643 ExecutionContext::new(session.clone(), timeout).into();
644 let query_manager = session.env().query_manager().clone();
645
646 query_manager
647 .schedule(execution_context, query, snapshot_override)
648 .await
649 .map_err(|err| err.into())
650}
651
652pub async fn local_execute(
653 session: Arc<SessionImpl>,
654 mut query: Query,
655 can_timeout_cancel: bool,
656 snapshot_override: Option<ReadSnapshot>,
657) -> Result<LocalQueryStream> {
658 let timeout = if cfg!(madsim) {
659 None
660 } else if can_timeout_cancel {
661 Some(session.statement_timeout())
662 } else {
663 None
664 };
665 let front_env = session.env();
666
667 let snapshot = snapshot_override.unwrap_or_else(|| session.pinned_snapshot());
668
669 snapshot.fill_batch_query_epoch(&mut query)?;
670
671 let execution = LocalQueryExecution::new(
672 query,
673 front_env.clone(),
674 snapshot.support_barrier_read(),
675 session,
676 timeout,
677 );
678
679 Ok(execution.stream_rows())
680}