risedev/
config.rs

1// Copyright 2025 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::path::Path;
17
18use anyhow::{Result, anyhow, bail};
19use itertools::Itertools;
20use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
21
22use crate::ServiceConfig;
23
24mod dollar_expander;
25mod id_expander;
26mod provide_expander;
27mod use_expander;
28use dollar_expander::DollarExpander;
29use id_expander::IdExpander;
30use provide_expander::ProvideExpander;
31use use_expander::UseExpander;
32
33/// The main configuration file name.
34pub const RISEDEV_CONFIG_FILE: &str = "risedev.yml";
35/// The extra user profiles file name.
36pub const RISEDEV_USER_PROFILES_FILE: &str = "risedev-profiles.user.yml";
37
38pub struct ConfigExpander;
39
40impl ConfigExpander {
41    /// Load a single document YAML file.
42    fn load_yaml(path: impl AsRef<Path>) -> Result<Yaml> {
43        let path = path.as_ref();
44        let content = fs_err::read_to_string(path)?;
45        let [config]: [_; 1] = YamlLoader::load_from_str(&content)?
46            .try_into()
47            .map_err(|_| anyhow!("expect `{}` to have only one section", path.display()))?;
48        Ok(config)
49    }
50
51    /// Transforms `risedev.yml` and `risedev-profiles.user.yml` to a fully expanded yaml file.
52    ///
53    /// Format:
54    ///
55    /// ```yaml
56    /// my-profile:
57    ///   config-path: src/config/ci-recovery.toml
58    ///   env:
59    ///     RUST_LOG: "info,risingwave_storage::hummock=off"
60    ///     ENABLE_PRETTY_LOG: "true"
61    ///   steps:
62    ///     - use: minio
63    ///     - use: sqlite
64    ///     - use: meta-node
65    ///       meta-backend: sqlite
66    ///     - use: compute-node
67    ///       parallelism: 1
68    ///     - use: frontend
69    /// ```
70    ///
71    /// # Arguments
72    ///
73    /// * `root` is the root directory of these YAML files.
74    /// * `profile` is the selected config profile called by `risedev dev <profile>`. It is one of
75    ///   the keys in the `profile` section.
76    ///
77    /// # Returns
78    ///
79    /// `(config_path, env, steps)`
80    pub fn expand(
81        root: impl AsRef<Path>,
82        profile: &str,
83    ) -> Result<(Option<String>, Vec<String>, Yaml)> {
84        Self::expand_with_extra_info(root, profile, HashMap::new())
85    }
86
87    /// See [`ConfigExpander::expand`] for other information.
88    ///
89    /// # Arguments
90    ///
91    /// - `extra_info` is additional variables for variable expansion by [`DollarExpander`].
92    pub fn expand_with_extra_info(
93        root: impl AsRef<Path>,
94        profile: &str,
95        extra_info: HashMap<String, String>,
96    ) -> Result<(Option<String>, Vec<String>, Yaml)> {
97        let global_path = root.as_ref().join(RISEDEV_CONFIG_FILE);
98        let global_yaml = Self::load_yaml(global_path)?;
99        let global_config = global_yaml
100            .as_hash()
101            .ok_or_else(|| anyhow!("expect config to be a hashmap"))?;
102
103        let all_profile_section = {
104            let mut all = global_config
105                .get(&Yaml::String("profile".to_owned()))
106                .ok_or_else(|| anyhow!("expect `profile` section"))?
107                .as_hash()
108                .ok_or_else(|| anyhow!("expect `profile` section to be a hashmap"))?
109                .to_owned();
110
111            // Add user profiles if exists.
112            let user_profiles_path = root.as_ref().join(RISEDEV_USER_PROFILES_FILE);
113            if user_profiles_path.is_file() {
114                let yaml = Self::load_yaml(user_profiles_path)?;
115                let map = yaml.as_hash().ok_or_else(|| {
116                    anyhow!("expect `{RISEDEV_USER_PROFILES_FILE}` to be a hashmap")
117                })?;
118                for (k, v) in map {
119                    if all.insert(k.clone(), v.clone()).is_some() {
120                        bail!(
121                            "find duplicated config key `{k:?}` in `{RISEDEV_USER_PROFILES_FILE}`"
122                        );
123                    }
124                }
125            }
126
127            all
128        };
129
130        let template_section = global_config
131            .get(&Yaml::String("template".to_owned()))
132            .ok_or_else(|| anyhow!("expect `profile` section"))?;
133
134        let profile_section = all_profile_section
135            .get(&Yaml::String(profile.to_owned()))
136            .ok_or_else(|| anyhow!("profile '{}' not found", profile))?
137            .as_hash()
138            .ok_or_else(|| anyhow!("expect `profile` section to be a hashmap"))?;
139
140        let config_path = profile_section
141            .get(&Yaml::String("config-path".to_owned()))
142            .and_then(|s| s.as_str())
143            .map(|s| s.to_owned());
144        let mut env = vec![];
145        if let Some(env_section) = profile_section.get(&Yaml::String("env".to_owned())) {
146            let env_section = env_section
147                .as_hash()
148                .ok_or_else(|| anyhow!("expect `env` section to be a hashmap"))?;
149
150            for (k, v) in env_section {
151                let key = k
152                    .as_str()
153                    .ok_or_else(|| anyhow!("expect env key to be a string"))?;
154                let value = v
155                    .as_str()
156                    .ok_or_else(|| anyhow!("expect env value to be a string"))?;
157                env.push(format!("{}={}", key, value));
158            }
159        }
160
161        let steps = profile_section
162            .get(&Yaml::String("steps".to_owned()))
163            .ok_or_else(|| anyhow!("expect `steps` section"))?
164            .clone();
165
166        let steps = UseExpander::new(template_section)?.visit(steps)?;
167        let steps = DollarExpander::new(extra_info).visit(steps)?;
168        let steps = IdExpander::new(&steps)?.visit(steps)?;
169        let steps = ProvideExpander::new(&steps)?.visit(steps)?;
170
171        Ok((config_path, env, steps))
172    }
173
174    /// Parses the expanded yaml into [`ServiceConfig`]s.
175    /// The order is the same as the original array's order.
176    pub fn deserialize(expanded_config: &Yaml) -> Result<Vec<ServiceConfig>> {
177        let steps = expanded_config
178            .as_vec()
179            .ok_or_else(|| anyhow!("expect steps to be an array"))?;
180        let config: Vec<ServiceConfig> = steps
181            .iter()
182            .map(|step| {
183                let use_type = step
184                    .as_hash()
185                    .ok_or_else(|| anyhow!("expect step to be a hashmap"))?;
186                let use_type = use_type
187                    .get(&Yaml::String("use".to_owned()))
188                    .ok_or_else(|| anyhow!("expect `use` in step"))?;
189                let use_type = use_type
190                    .as_str()
191                    .ok_or_else(|| anyhow!("expect `use` to be a string"))?
192                    .to_owned();
193                let mut out_str = String::new();
194                let mut emitter = YamlEmitter::new(&mut out_str);
195                emitter.dump(step)?;
196                let result = match use_type.as_str() {
197                    "minio" => ServiceConfig::Minio(serde_yaml::from_str(&out_str)?),
198                    "sqlite" => ServiceConfig::Sqlite(serde_yaml::from_str(&out_str)?),
199                    "frontend" => ServiceConfig::Frontend(serde_yaml::from_str(&out_str)?),
200                    "compactor" => ServiceConfig::Compactor(serde_yaml::from_str(&out_str)?),
201                    "compute-node" => ServiceConfig::ComputeNode(serde_yaml::from_str(&out_str)?),
202                    "meta-node" => ServiceConfig::MetaNode(serde_yaml::from_str(&out_str)?),
203                    "prometheus" => ServiceConfig::Prometheus(serde_yaml::from_str(&out_str)?),
204                    "grafana" => ServiceConfig::Grafana(serde_yaml::from_str(&out_str)?),
205                    "tempo" => ServiceConfig::Tempo(serde_yaml::from_str(&out_str)?),
206                    "opendal" => ServiceConfig::Opendal(serde_yaml::from_str(&out_str)?),
207                    "aws-s3" => ServiceConfig::AwsS3(serde_yaml::from_str(&out_str)?),
208                    "kafka" => ServiceConfig::Kafka(serde_yaml::from_str(&out_str)?),
209                    "pubsub" => ServiceConfig::Pubsub(serde_yaml::from_str(&out_str)?),
210                    "redis" => ServiceConfig::Redis(serde_yaml::from_str(&out_str)?),
211                    "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?),
212                    "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?),
213                    "postgres" => ServiceConfig::Postgres(serde_yaml::from_str(&out_str)?),
214                    "sqlserver" => ServiceConfig::SqlServer(serde_yaml::from_str(&out_str)?),
215                    "schema-registry" => {
216                        ServiceConfig::SchemaRegistry(serde_yaml::from_str(&out_str)?)
217                    }
218                    other => return Err(anyhow!("unsupported use type: {}", other)),
219                };
220                Ok(result)
221            })
222            .try_collect()?;
223
224        let mut services = HashMap::new();
225        for x in &config {
226            let id = x.id().to_owned();
227            if services.insert(id.clone(), x).is_some() {
228                return Err(anyhow!("duplicate id: {}", id));
229            }
230        }
231        Ok(config)
232    }
233}
risedev/config.rs

risedev/
config.rs
