Skip to main content

risedev/task/
mod.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod clickhouse_service;
16mod compactor_service;
17mod compute_node_service;
18mod configure_tmux_service;
19mod docker_service;
20mod doris_service;
21mod dummy_service;
22mod elasticsearch_service;
23mod ensure_stop_service;
24mod frontend_service;
25mod grafana_service;
26mod kafka_service;
27mod lakekeeper_service;
28mod mariadb_service;
29mod meta_node_service;
30mod minio_service;
31mod moat_service;
32mod mongodb_service;
33mod moto_service;
34mod mqtt_service;
35mod mysql_service;
36mod nats_service;
37mod opensearch_service;
38mod oracle_service;
39mod postgres_service;
40mod prometheus_service;
41mod pubsub_service;
42mod pulsar_service;
43mod redis_service;
44mod schema_registry_service;
45mod sql_server_service;
46mod starrocks_service;
47mod task_clickhouse_ready_check;
48mod task_configure_minio;
49mod task_db_ready_check;
50mod task_doris_ready_check;
51mod task_kafka_ready_check;
52mod task_log_ready_check;
53mod task_pubsub_emu_ready_check;
54mod task_redis_ready_check;
55mod task_starrocks_ready_check;
56mod task_tcp_ready_check;
57mod tempo_service;
58mod utils;
59
60use std::collections::HashMap;
61use std::env;
62use std::io::Write;
63use std::net::{TcpStream, ToSocketAddrs};
64use std::path::{Path, PathBuf};
65use std::process::{Command, Output};
66use std::sync::Arc;
67use std::time::Duration;
68
69use anyhow::{Context, Result, anyhow};
70use indicatif::ProgressBar;
71use reqwest::blocking::{Client, Response};
72use tempfile::TempDir;
73pub use utils::*;
74
75pub use self::clickhouse_service::*;
76pub use self::compactor_service::*;
77pub use self::compute_node_service::*;
78pub use self::configure_tmux_service::*;
79pub use self::doris_service::*;
80pub use self::dummy_service::DummyService;
81pub use self::elasticsearch_service::*;
82pub use self::ensure_stop_service::*;
83pub use self::frontend_service::*;
84pub use self::grafana_service::*;
85pub use self::kafka_service::*;
86pub use self::lakekeeper_service::*;
87pub use self::mariadb_service::*;
88pub use self::meta_node_service::*;
89pub use self::minio_service::*;
90pub use self::moat_service::*;
91pub use self::mongodb_service::*;
92pub use self::moto_service::*;
93pub use self::mqtt_service::*;
94pub use self::mysql_service::*;
95pub use self::nats_service::*;
96pub use self::opensearch_service::*;
97pub use self::oracle_service::*;
98pub use self::postgres_service::*;
99pub use self::prometheus_service::*;
100pub use self::pubsub_service::*;
101pub use self::pulsar_service::*;
102pub use self::redis_service::*;
103pub use self::schema_registry_service::SchemaRegistryService;
104pub use self::sql_server_service::*;
105pub use self::starrocks_service::*;
106pub use self::task_clickhouse_ready_check::*;
107pub use self::task_configure_minio::*;
108pub use self::task_db_ready_check::*;
109pub use self::task_doris_ready_check::*;
110pub use self::task_kafka_ready_check::*;
111pub use self::task_log_ready_check::*;
112pub use self::task_pubsub_emu_ready_check::*;
113pub use self::task_redis_ready_check::*;
114pub use self::task_starrocks_ready_check::*;
115pub use self::task_tcp_ready_check::*;
116pub use self::tempo_service::*;
117use crate::util::{begin_spin, complete_spin, get_program_args, get_program_name};
118use crate::wait::{wait, wait_tcp_available};
119
120pub trait Task: 'static + Send {
121    /// Execute the task
122    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()>;
123
124    /// Get task id used in progress bar
125    fn id(&self) -> String {
126        "<task>".into()
127    }
128}
129
130/// A context used in task execution
131pub struct ExecuteContext<W>
132where
133    W: std::io::Write,
134{
135    /// Global log file object. (aka. risedev.log)
136    pub log: W,
137
138    /// Progress bar on screen.
139    pub pb: ProgressBar,
140
141    /// The directory for checking status.
142    ///
143    /// `RiseDev` will instruct every task to output their status to a file in temporary folder. By
144    /// checking this file, we can know whether a task has early exited.
145    pub status_dir: Arc<TempDir>,
146
147    /// The current service id running in this context.
148    pub id: Option<String>,
149
150    /// The status file corresponding to the current context.
151    pub status_file: Option<PathBuf>,
152
153    /// The log file corresponding to the current context. (e.g. frontend-4566.log)
154    pub log_file: Option<PathBuf>,
155}
156
157impl<W> ExecuteContext<W>
158where
159    W: std::io::Write,
160{
161    pub fn new(log: W, pb: ProgressBar, status_dir: Arc<TempDir>) -> Self {
162        Self {
163            log,
164            pb,
165            status_dir,
166            status_file: None,
167            log_file: None,
168            id: None,
169        }
170    }
171
172    pub fn service(&mut self, task: &impl Task) {
173        let id = task.id();
174        if !id.is_empty() {
175            begin_spin(&self.pb);
176            self.pb.set_prefix(id.clone());
177            self.id = Some(id.clone());
178            self.status_file = Some(self.status_dir.path().join(format!("{}.status", id)));
179
180            // Remove the old log file if exists to avoid confusion.
181            let log_file = Path::new(&env::var("PREFIX_LOG").unwrap())
182                .join(format!("{}.log", self.id.as_ref().unwrap()));
183            fs_err::remove_file(&log_file).ok();
184            self.log_file = Some(log_file);
185        }
186    }
187
188    pub fn run_command(&mut self, mut cmd: Command) -> Result<Output> {
189        let program_name = get_program_name(&cmd);
190
191        writeln!(self.log, "> {} {}", program_name, get_program_args(&cmd))?;
192
193        // Record a service command for `risedev restart`
194        let node_id = self.id.as_ref();
195        if program_name == "tmux"
196            && let Some(node_id) = node_id
197            && node_id != "tmux-configure"
198        {
199            let cmd: HashMap<String, String> = HashMap::from_iter([(
200                node_id.clone(),
201                format!("{} {}", program_name, get_program_args(&cmd)),
202            )]);
203            let prefix_config = env::var("PREFIX_CONFIG").unwrap();
204            let path = Path::new(&prefix_config).join("risedev_commands.yaml");
205            let content = serde_yaml::to_string(&cmd)?;
206            fs_err::OpenOptions::new()
207                .create(true)
208                .append(true)
209                .open(path)?
210                .write_all(content.as_bytes())?;
211        }
212
213        let output = cmd.output()?;
214
215        let mut full_output = String::from_utf8_lossy(&output.stdout).to_string();
216        full_output.extend(String::from_utf8_lossy(&output.stderr).chars());
217
218        write!(self.log, "{}", full_output)?;
219
220        writeln!(
221            self.log,
222            "({} exited with {:?})",
223            program_name,
224            output.status.code()
225        )?;
226
227        writeln!(self.log, "---")?;
228
229        output.status.exit_ok().context(full_output)?;
230
231        Ok(output)
232    }
233
234    pub fn complete_spin(&mut self) {
235        complete_spin(&self.pb);
236    }
237
238    pub fn status_path(&self) -> PathBuf {
239        self.status_file.clone().unwrap()
240    }
241
242    pub fn log_path(&self) -> &Path {
243        self.log_file.as_ref().unwrap().as_path()
244    }
245
246    pub fn wait_tcp(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
247        let addr = server
248            .as_ref()
249            .to_socket_addrs()?
250            .next()
251            .with_context(|| format!("failed to resolve {}", server.as_ref()))?;
252        wait(
253            || {
254                TcpStream::connect_timeout(&addr, Duration::from_secs(1)).with_context(|| {
255                    format!("failed to establish tcp connection to {}", server.as_ref())
256                })?;
257                Ok(())
258            },
259            &mut self.log,
260            self.status_file.as_ref().unwrap(),
261            self.id.as_ref().unwrap(),
262            Some(Duration::from_secs(30)),
263            true,
264        )?;
265        Ok(())
266    }
267
268    fn wait_http_with_response_cb(
269        &mut self,
270        server: impl AsRef<str>,
271        cb: impl Fn(Response) -> anyhow::Result<()>,
272    ) -> anyhow::Result<()> {
273        let server = server.as_ref();
274        wait(
275            || {
276                let resp = Client::new()
277                    .get(server)
278                    .timeout(Duration::from_secs(1))
279                    .body("")
280                    .send()?
281                    .error_for_status()
282                    .with_context(|| {
283                        format!("failed to establish http connection to {}", server)
284                    })?;
285
286                cb(resp)
287            },
288            &mut self.log,
289            self.status_file.as_ref().unwrap(),
290            self.id.as_ref().unwrap(),
291            Some(Duration::from_secs(30)),
292            true,
293        )
294    }
295
296    pub fn wait_http(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
297        self.wait_http_with_response_cb(server, |_| Ok(()))
298    }
299
300    pub fn wait_http_with_text_cb(
301        &mut self,
302        server: impl AsRef<str>,
303        cb: impl Fn(&str) -> bool,
304    ) -> anyhow::Result<()> {
305        self.wait_http_with_response_cb(server, |resp| {
306            let data = resp.text()?;
307            if cb(&data) {
308                Ok(())
309            } else {
310                Err(anyhow!(
311                    "http health check callback failed with body: {:?}",
312                    data
313                ))
314            }
315        })
316    }
317
318    pub fn wait(&mut self, wait_func: impl FnMut() -> Result<()>) -> anyhow::Result<()> {
319        wait(
320            wait_func,
321            &mut self.log,
322            self.status_file.as_ref().unwrap(),
323            self.id.as_ref().unwrap(),
324            Some(Duration::from_secs(30)),
325            true,
326        )
327    }
328
329    /// Wait for a TCP port to close
330    pub fn wait_tcp_close(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
331        wait_tcp_available(server, Some(Duration::from_secs(30)))?;
332        Ok(())
333    }
334
335    /// Wait for a user-managed service to be available
336    pub fn wait_tcp_user(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
337        let addr = server
338            .as_ref()
339            .to_socket_addrs()?
340            .next()
341            .unwrap_or_else(|| panic!("failed to resolve {}", server.as_ref()));
342        wait(
343            || {
344                TcpStream::connect_timeout(&addr, Duration::from_secs(1))?;
345                Ok(())
346            },
347            &mut self.log,
348            self.status_file.as_ref().unwrap(),
349            self.id.as_ref().unwrap(),
350            None,
351            false,
352        )?;
353        Ok(())
354    }
355
356    pub fn tmux_run(&self, user_cmd: Command) -> anyhow::Result<Command> {
357        let prefix_path = env::var("PREFIX_BIN")?;
358        let mut cmd = new_tmux_command();
359        cmd.arg("new-window")
360            // Set target name
361            .arg("-t")
362            .arg(RISEDEV_NAME)
363            // Switch to background window
364            .arg("-d")
365            // Set session name for this window
366            .arg("-n")
367            .arg(self.id.as_ref().unwrap());
368
369        if let Some(dir) = user_cmd.get_current_dir() {
370            cmd.arg("-c").arg(dir);
371        }
372        for (k, v) in user_cmd.get_envs() {
373            cmd.arg("-e");
374            if let Some(v) = v {
375                cmd.arg(format!("{}={}", k.to_string_lossy(), v.to_string_lossy()));
376            } else {
377                cmd.arg(k);
378            }
379        }
380        cmd.arg(Path::new(&prefix_path).join("run_command.sh"));
381        cmd.arg(self.log_path());
382        cmd.arg(self.status_path());
383        cmd.arg(user_cmd.get_program());
384        for arg in user_cmd.get_args() {
385            cmd.arg(arg);
386        }
387
388        Ok(cmd)
389    }
390}