Skip to main content

risingwave_frontend/handler/
query.rs

1// Copyright 2022 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashSet;
16use std::sync::Arc;
17use std::time::Instant;
18
19use itertools::Itertools;
20use pgwire::pg_field_descriptor::PgFieldDescriptor;
21use pgwire::pg_response::{PgResponse, StatementType};
22use pgwire::types::Format;
23use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
24use risingwave_common::bail_not_implemented;
25use risingwave_common::catalog::{FunctionId, Schema, SecretId};
26use risingwave_common::id::ObjectId;
27use risingwave_common::session_config::QueryMode;
28use risingwave_common::types::{DataType, Datum};
29use risingwave_sqlparser::ast::{SetExpr, Statement};
30
31use super::extended_handle::{PortalResult, PrepareStatement, PreparedResult};
32use super::{PgResponseStream, RwPgResponse, create_mv, declare_cursor};
33use crate::binder::{Binder, BoundCreateView, BoundStatement};
34#[cfg(feature = "datafusion")]
35use crate::datafusion::DfBatchQueryPlanResult;
36use crate::error::{ErrorCode, Result, RwError};
37use crate::handler::HandlerArgs;
38use crate::handler::flush::do_flush;
39use crate::handler::util::{DataChunkToRowSetAdapter, to_pg_field};
40use crate::optimizer::plan_node::{BatchPlanRef, Explain};
41use crate::optimizer::{
42    BatchPlanRoot, ExecutionModeDecider, OptimizerContext, OptimizerContextRef, SysTableVisitor,
43};
44use crate::planner::Planner;
45use crate::scheduler::plan_fragmenter::Query;
46use crate::scheduler::{
47    BatchPlanFragmenter, DistributedQueryStream, ExecutionContext, ExecutionContextRef,
48    LocalQueryExecution, LocalQueryStream, ReadSnapshot,
49};
50use crate::session::SessionImpl;
51
52/// Choice between running RisingWave's own batch executor (Rw) or a `DataFusion` (DF) logical plan.
53pub enum BatchPlanChoice {
54    Rw(RwBatchQueryPlanResult),
55    #[cfg(feature = "datafusion")]
56    Df(DfBatchQueryPlanResult),
57}
58
59impl BatchPlanChoice {
60    pub fn unwrap_rw(self) -> Result<RwBatchQueryPlanResult> {
61        match self {
62            BatchPlanChoice::Rw(result) => Ok(result),
63            #[cfg(feature = "datafusion")]
64            BatchPlanChoice::Df { .. } => {
65                risingwave_common::bail!(
66                    "Expected RisingWave plan in BatchPlanChoice, but got DataFusion plan"
67                )
68            }
69        }
70    }
71}
72
73pub async fn handle_query(
74    handler_args: HandlerArgs,
75    stmt: Statement,
76    formats: Vec<Format>,
77) -> Result<RwPgResponse> {
78    let session = handler_args.session.clone();
79    let context = OptimizerContext::from_handler_args(handler_args)?;
80
81    #[cfg(feature = "datafusion")]
82    {
83        // We construct a future manually here to make sure this async function is `Send`.
84        // `BatchPlanChoice` is non-Send, and rust cannot prove it has dropped before await point.
85        // See more details in https://github.com/rust-lang/rust/issues/128095
86        use futures::FutureExt;
87
88        use crate::datafusion::execute_datafusion_plan;
89
90        let future = match gen_batch_plan_by_statement(&session, context.into(), stmt)? {
91            BatchPlanChoice::Rw(plan_result) => {
92                let plan_fragmenter_result = risingwave_expr::expr_context::TIME_ZONE::sync_scope(
93                    session.config().timezone(),
94                    || gen_batch_plan_fragmenter(&session, plan_result),
95                )?;
96                execute_risingwave_plan(session, plan_fragmenter_result, formats).left_future()
97            }
98            BatchPlanChoice::Df(plan_result) => {
99                execute_datafusion_plan(session, plan_result, formats).right_future()
100            }
101        };
102        future.await
103    }
104
105    #[cfg(not(feature = "datafusion"))]
106    {
107        let future = match gen_batch_plan_by_statement(&session, context.into(), stmt)? {
108            BatchPlanChoice::Rw(plan_result) => {
109                let plan_fragmenter_result = risingwave_expr::expr_context::TIME_ZONE::sync_scope(
110                    session.config().timezone(),
111                    || gen_batch_plan_fragmenter(&session, plan_result),
112                )?;
113                execute_risingwave_plan(session, plan_fragmenter_result, formats)
114            }
115        };
116        future.await
117    }
118}
119
120fn handle_parse_inner(binder: Binder, statement: Statement) -> Result<PrepareStatement> {
121    let bound_result = gen_bound(binder, statement.clone())?;
122
123    Ok(PrepareStatement::Prepared(PreparedResult {
124        statement,
125        bound_result,
126    }))
127}
128
129pub fn handle_parse_for_batch(
130    handler_args: HandlerArgs,
131    statement: Statement,
132    specified_param_types: Vec<Option<DataType>>,
133) -> Result<PrepareStatement> {
134    let binder = Binder::new_for_batch(&handler_args.session)
135        .with_specified_params_types(specified_param_types);
136    handle_parse_inner(binder, statement)
137}
138
139pub fn handle_parse_for_stream(
140    handler_args: HandlerArgs,
141    statement: Statement,
142    specified_param_types: Vec<Option<DataType>>,
143) -> Result<PrepareStatement> {
144    let binder = Binder::new_for_stream(&handler_args.session)
145        .with_specified_params_types(specified_param_types);
146    handle_parse_inner(binder, statement)
147}
148
149/// Execute a "Portal", which is a prepared statement with bound parameters.
150pub async fn handle_execute(
151    handler_args: HandlerArgs,
152    portal: PortalResult,
153) -> Result<RwPgResponse> {
154    let PortalResult {
155        bound_result,
156        result_formats,
157        statement,
158    } = portal;
159    match statement {
160        Statement::Query(_)
161        | Statement::Insert { .. }
162        | Statement::Delete { .. }
163        | Statement::Update { .. } => {
164            // Execute a batch query
165            let session = handler_args.session.clone();
166            let plan_fragmenter_result = {
167                let context = OptimizerContext::from_handler_args(handler_args)?;
168                let plan_result =
169                    gen_batch_query_plan(&session, context.into(), bound_result)?.unwrap_rw()?;
170                // Time zone is used by Hummock time travel query.
171                risingwave_expr::expr_context::TIME_ZONE::sync_scope(
172                    session.config().timezone(),
173                    || gen_batch_plan_fragmenter(&session, plan_result),
174                )?
175            };
176            execute_risingwave_plan(session, plan_fragmenter_result, result_formats).await
177        }
178        Statement::CreateView { materialized, .. } if materialized => {
179            // Execute a CREATE MATERIALIZED VIEW
180            let BoundResult {
181                bound,
182                dependent_relations,
183                dependent_udfs,
184                dependent_secrets,
185                ..
186            } = bound_result;
187            let create_mv = if let BoundStatement::CreateView(create_mv) = bound {
188                create_mv
189            } else {
190                unreachable!("expect a BoundStatement::CreateView")
191            };
192            let BoundCreateView {
193                or_replace,
194                materialized: _,
195                if_not_exists,
196                name,
197                columns,
198                query,
199                emit_mode,
200                with_options,
201            } = *create_mv;
202            if or_replace {
203                bail_not_implemented!("CREATE OR REPLACE VIEW");
204            }
205
206            // Hack: replace the `with_options` with the bounded ones.
207            let handler_args = HandlerArgs {
208                session: handler_args.session.clone(),
209                sql: handler_args.sql.clone(),
210                normalized_sql: handler_args.normalized_sql.clone(),
211                with_options: crate::WithOptions::try_from(with_options.as_slice())?,
212            };
213
214            create_mv::handle_create_mv_bound(
215                handler_args,
216                if_not_exists,
217                name,
218                *query,
219                dependent_relations,
220                dependent_udfs,
221                dependent_secrets,
222                columns,
223                emit_mode,
224            )
225            .await
226        }
227        Statement::DeclareCursor { stmt } => match stmt.declare_cursor {
228            risingwave_sqlparser::ast::DeclareCursor::Query(_) => {
229                let session = handler_args.session.clone();
230                let plan_fragmenter_result = {
231                    let context = OptimizerContext::from_handler_args(handler_args.clone())?;
232                    let plan_result = gen_batch_query_plan(&session, context.into(), bound_result)?
233                        .unwrap_rw()?;
234                    gen_batch_plan_fragmenter(&session, plan_result)?
235                };
236                declare_cursor::handle_bound_declare_query_cursor(
237                    handler_args,
238                    stmt.cursor_name,
239                    plan_fragmenter_result,
240                )
241                .await
242            }
243            risingwave_sqlparser::ast::DeclareCursor::Subscription(sub_name, rw_timestamp) => {
244                declare_cursor::handle_declare_subscription_cursor(
245                    handler_args,
246                    sub_name,
247                    stmt.cursor_name,
248                    rw_timestamp,
249                )
250                .await
251            }
252        },
253        _ => unreachable!(),
254    }
255}
256
257pub fn gen_batch_plan_by_statement(
258    session: &SessionImpl,
259    context: OptimizerContextRef,
260    stmt: Statement,
261) -> Result<BatchPlanChoice> {
262    let binder = Binder::new_for_batch(session);
263    let bound_result = gen_bound(binder, stmt)?;
264    gen_batch_query_plan(session, context, bound_result)
265}
266
267#[derive(Clone)]
268pub struct BoundResult {
269    pub(crate) stmt_type: StatementType,
270    pub(crate) must_dist: bool,
271    pub(crate) bound: BoundStatement,
272    pub(crate) param_types: Vec<DataType>,
273    pub(crate) parsed_params: Option<Vec<Datum>>,
274    pub(crate) dependent_relations: HashSet<ObjectId>,
275    /// TODO(rc): merge with `dependent_relations`
276    pub(crate) dependent_udfs: HashSet<FunctionId>,
277    pub(crate) dependent_secrets: HashSet<SecretId>,
278}
279
280fn gen_bound(mut binder: Binder, stmt: Statement) -> Result<BoundResult> {
281    let stmt_type = StatementType::infer_from_statement(&stmt)
282        .map_err(|err| RwError::from(ErrorCode::InvalidInputSyntax(err)))?;
283    let must_dist = must_run_in_distributed_mode(&stmt)?;
284
285    let bound = binder.bind(stmt)?;
286
287    Ok(BoundResult {
288        stmt_type,
289        must_dist,
290        bound,
291        param_types: binder.export_param_types()?,
292        parsed_params: None,
293        dependent_relations: binder.included_relations().clone(),
294        dependent_udfs: binder.included_udfs().clone(),
295        dependent_secrets: binder.included_secrets().clone(),
296    })
297}
298
299pub struct RwBatchQueryPlanResult {
300    pub(crate) plan: BatchPlanRef,
301    pub(crate) query_mode: QueryMode,
302    pub(crate) schema: Schema,
303    pub(crate) stmt_type: StatementType,
304}
305
306fn gen_batch_query_plan(
307    session: &SessionImpl,
308    context: OptimizerContextRef,
309    bind_result: BoundResult,
310) -> Result<BatchPlanChoice> {
311    let BoundResult {
312        stmt_type,
313        must_dist,
314        bound,
315        ..
316    } = bind_result;
317
318    let mut planner = if matches!(bound, BoundStatement::Query(_)) {
319        Planner::new_for_batch_dql(context)
320    } else {
321        Planner::new_for_batch(context)
322    };
323
324    let logical = planner.plan(bound)?;
325    let schema = logical.schema();
326    let optimized_logical = logical.gen_optimized_logical_plan_for_batch()?;
327
328    #[cfg(feature = "datafusion")]
329    {
330        if session.config().enable_datafusion_engine() {
331            use thiserror_ext::AsReport;
332
333            use crate::datafusion::{GenDataFusionPlanError, try_gen_datafusion_plan};
334
335            match try_gen_datafusion_plan(&optimized_logical) {
336                Ok(plan) => {
337                    return Ok(BatchPlanChoice::Df(DfBatchQueryPlanResult {
338                        plan,
339                        schema,
340                        stmt_type,
341                    }));
342                }
343                Err(GenDataFusionPlanError::MissingIcebergScan) => {}
344                Err(err) => {
345                    tracing::warn!(
346                        "Failed to generate DataFusion plan, fallback to RisingWave plan: {}",
347                        err.as_report()
348                    );
349                }
350            }
351        }
352    }
353
354    let batch_plan = optimized_logical.gen_batch_plan()?;
355
356    let must_local = must_run_in_local_mode(&batch_plan);
357
358    let query_mode = match (must_dist, must_local) {
359        (true, true) => {
360            return Err(ErrorCode::InternalError(
361                "the query is forced to both local and distributed mode by optimizer".to_owned(),
362            )
363            .into());
364        }
365        (true, false) => QueryMode::Distributed,
366        (false, true) => QueryMode::Local,
367        (false, false) => match session.config().query_mode() {
368            QueryMode::Auto => determine_query_mode(&batch_plan),
369            QueryMode::Local => QueryMode::Local,
370            QueryMode::Distributed => QueryMode::Distributed,
371        },
372    };
373
374    let physical = match query_mode {
375        QueryMode::Auto => unreachable!(),
376        QueryMode::Local => batch_plan.gen_batch_local_plan()?,
377        QueryMode::Distributed => batch_plan.gen_batch_distributed_plan()?,
378    };
379
380    let result = RwBatchQueryPlanResult {
381        plan: physical,
382        query_mode,
383        schema,
384        stmt_type,
385    };
386    Ok(BatchPlanChoice::Rw(result))
387}
388
389fn must_run_in_distributed_mode(stmt: &Statement) -> Result<bool> {
390    fn is_insert_using_select(stmt: &Statement) -> bool {
391        fn has_select_query(set_expr: &SetExpr) -> bool {
392            match set_expr {
393                SetExpr::Select(_) => true,
394                SetExpr::Query(query) => has_select_query(&query.body),
395                SetExpr::SetOperation { left, right, .. } => {
396                    has_select_query(left) || has_select_query(right)
397                }
398                SetExpr::Values(_) => false,
399            }
400        }
401
402        matches!(
403            stmt,
404            Statement::Insert {source, ..} if has_select_query(&source.body)
405        )
406    }
407
408    let stmt_type = StatementType::infer_from_statement(stmt)
409        .map_err(|err| RwError::from(ErrorCode::InvalidInputSyntax(err)))?;
410
411    Ok(matches!(
412        stmt_type,
413        StatementType::UPDATE
414            | StatementType::DELETE
415            | StatementType::UPDATE_RETURNING
416            | StatementType::DELETE_RETURNING
417    ) | is_insert_using_select(stmt))
418}
419
420fn must_run_in_local_mode(batch_plan: &BatchPlanRoot) -> bool {
421    SysTableVisitor::has_sys_table(batch_plan)
422}
423
424fn determine_query_mode(batch_plan: &BatchPlanRoot) -> QueryMode {
425    if ExecutionModeDecider::run_in_local_mode(batch_plan) {
426        QueryMode::Local
427    } else {
428        QueryMode::Distributed
429    }
430}
431
432pub struct BatchPlanFragmenterResult {
433    pub(crate) plan_fragmenter: BatchPlanFragmenter,
434    pub(crate) query_mode: QueryMode,
435    pub(crate) schema: Schema,
436    pub(crate) stmt_type: StatementType,
437    /// Per-query override, used to keep subscription snapshots pinned through execution.
438    pub(crate) snapshot_override: Option<ReadSnapshot>,
439}
440
441pub fn gen_batch_plan_fragmenter(
442    session: &SessionImpl,
443    plan_result: RwBatchQueryPlanResult,
444) -> Result<BatchPlanFragmenterResult> {
445    gen_batch_plan_fragmenter_with_snapshot(session, plan_result, None)
446}
447
448pub fn gen_batch_plan_fragmenter_with_snapshot(
449    session: &SessionImpl,
450    plan_result: RwBatchQueryPlanResult,
451    snapshot_override: Option<ReadSnapshot>,
452) -> Result<BatchPlanFragmenterResult> {
453    let RwBatchQueryPlanResult {
454        plan,
455        query_mode,
456        schema,
457        stmt_type,
458        ..
459    } = plan_result;
460
461    tracing::trace!(
462        "Generated query plan: {:?}, query_mode:{:?}",
463        plan.explain_to_string(),
464        query_mode
465    );
466    let worker_node_manager_reader = WorkerNodeSelector::new(
467        session.env().worker_node_manager_ref(),
468        snapshot_override
469            .as_ref()
470            .or(session.get_pinned_snapshot().as_ref())
471            .map_or_else(
472                || session.is_barrier_read(),
473                ReadSnapshot::support_barrier_read,
474            ),
475    );
476    let plan_fragmenter = BatchPlanFragmenter::new(
477        worker_node_manager_reader,
478        session.env().catalog_reader().clone(),
479        session.config().batch_parallelism().0,
480        plan,
481    )?;
482
483    Ok(BatchPlanFragmenterResult {
484        plan_fragmenter,
485        query_mode,
486        schema,
487        stmt_type,
488        snapshot_override,
489    })
490}
491
492pub async fn create_stream(
493    session: Arc<SessionImpl>,
494    plan_fragmenter_result: BatchPlanFragmenterResult,
495    formats: Vec<Format>,
496) -> Result<(PgResponseStream, Vec<PgFieldDescriptor>)> {
497    let BatchPlanFragmenterResult {
498        plan_fragmenter,
499        query_mode,
500        schema,
501        stmt_type,
502        snapshot_override,
503    } = plan_fragmenter_result;
504
505    let mut can_timeout_cancel = true;
506    // Acquire the write guard for DML statements.
507    match stmt_type {
508        StatementType::INSERT
509        | StatementType::INSERT_RETURNING
510        | StatementType::DELETE
511        | StatementType::DELETE_RETURNING
512        | StatementType::UPDATE
513        | StatementType::UPDATE_RETURNING => {
514            session.txn_write_guard()?;
515            can_timeout_cancel = false;
516        }
517        _ => {}
518    }
519
520    let query = plan_fragmenter.generate_complete_query().await?;
521    tracing::trace!("Generated query after plan fragmenter: {:?}", &query);
522
523    let pg_descs = schema
524        .fields()
525        .iter()
526        .map(to_pg_field)
527        .collect::<Vec<PgFieldDescriptor>>();
528    let column_types = schema.fields().iter().map(|f| f.data_type()).collect_vec();
529
530    let row_stream = match query_mode {
531        QueryMode::Auto => unreachable!(),
532        QueryMode::Local => PgResponseStream::LocalQuery(DataChunkToRowSetAdapter::new(
533            local_execute(
534                session.clone(),
535                query,
536                can_timeout_cancel,
537                snapshot_override,
538            )
539            .await?,
540            column_types,
541            formats,
542            session.clone(),
543        )),
544        // Local mode do not support cancel tasks.
545        QueryMode::Distributed => {
546            PgResponseStream::DistributedQuery(DataChunkToRowSetAdapter::new(
547                distribute_execute(
548                    session.clone(),
549                    query,
550                    can_timeout_cancel,
551                    snapshot_override,
552                )
553                .await?,
554                column_types,
555                formats,
556                session.clone(),
557            ))
558        }
559    };
560
561    Ok((row_stream, pg_descs))
562}
563
564async fn execute_risingwave_plan(
565    session: Arc<SessionImpl>,
566    plan_fragmenter_result: BatchPlanFragmenterResult,
567    formats: Vec<Format>,
568) -> Result<RwPgResponse> {
569    // Used in counting row count.
570    let first_field_format = formats.first().copied().unwrap_or(Format::Text);
571    let query_mode = plan_fragmenter_result.query_mode;
572    let stmt_type = plan_fragmenter_result.stmt_type;
573
574    let query_start_time = Instant::now();
575    let (row_stream, pg_descs) =
576        create_stream(session.clone(), plan_fragmenter_result, formats).await?;
577
578    // We need to do some post work after the query is finished and before the `Complete` response
579    // it sent. This is achieved by the `callback` in `PgResponse`.
580    let callback = async move {
581        // Implicitly flush the writes.
582        if session.config().implicit_flush() && stmt_type.is_dml() {
583            do_flush(&session).await?;
584        }
585
586        // update some metrics
587        match query_mode {
588            QueryMode::Auto => unreachable!(),
589            QueryMode::Local => {
590                session
591                    .env()
592                    .frontend_metrics
593                    .latency_local_execution
594                    .observe(query_start_time.elapsed().as_secs_f64());
595
596                session
597                    .env()
598                    .frontend_metrics
599                    .query_counter_local_execution
600                    .inc();
601            }
602            QueryMode::Distributed => {
603                session
604                    .env()
605                    .query_manager()
606                    .query_metrics
607                    .query_latency
608                    .observe(query_start_time.elapsed().as_secs_f64());
609
610                session
611                    .env()
612                    .query_manager()
613                    .query_metrics
614                    .completed_query_counter
615                    .inc();
616            }
617        }
618
619        Ok(())
620    };
621
622    Ok(PgResponse::builder(stmt_type)
623        .row_cnt_format_opt(Some(first_field_format))
624        .values(row_stream, pg_descs)
625        .callback(callback)
626        .into())
627}
628
629pub async fn distribute_execute(
630    session: Arc<SessionImpl>,
631    query: Query,
632    can_timeout_cancel: bool,
633    snapshot_override: Option<ReadSnapshot>,
634) -> Result<DistributedQueryStream> {
635    let timeout = if cfg!(madsim) {
636        None
637    } else if can_timeout_cancel {
638        Some(session.statement_timeout())
639    } else {
640        None
641    };
642    let execution_context: ExecutionContextRef =
643        ExecutionContext::new(session.clone(), timeout).into();
644    let query_manager = session.env().query_manager().clone();
645
646    query_manager
647        .schedule(execution_context, query, snapshot_override)
648        .await
649        .map_err(|err| err.into())
650}
651
652pub async fn local_execute(
653    session: Arc<SessionImpl>,
654    mut query: Query,
655    can_timeout_cancel: bool,
656    snapshot_override: Option<ReadSnapshot>,
657) -> Result<LocalQueryStream> {
658    let timeout = if cfg!(madsim) {
659        None
660    } else if can_timeout_cancel {
661        Some(session.statement_timeout())
662    } else {
663        None
664    };
665    let front_env = session.env();
666
667    let snapshot = snapshot_override.unwrap_or_else(|| session.pinned_snapshot());
668
669    snapshot.fill_batch_query_epoch(&mut query)?;
670
671    let execution = LocalQueryExecution::new(
672        query,
673        front_env.clone(),
674        snapshot.support_barrier_read(),
675        session,
676        timeout,
677    );
678
679    Ok(execution.stream_rows())
680}