risedev/
risedev_env.rs

1// Copyright 2023 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(clippy::doc_markdown)] // RiseDev
16
17use std::fmt::Write;
18use std::process::Command;
19
20use crate::{Application, HummockInMemoryStrategy, ServiceConfig, add_hummock_backend};
21
22/// Generate environment variables (put in file `.risingwave/config/risedev-env`)
23/// from the given service configurations to be used by future
24/// RiseDev commands, like `risedev ctl` or `risedev psql` ().
25pub fn generate_risedev_env(services: &Vec<ServiceConfig>) -> String {
26    let mut env = String::new();
27    for item in services {
28        match item {
29            ServiceConfig::ComputeNode(c) => {
30                // RW_HUMMOCK_URL
31                // If the cluster is launched without a shared storage, we will skip this.
32                {
33                    let mut cmd = Command::new("compute-node");
34                    if add_hummock_backend(
35                        "dummy",
36                        c.provide_opendal.as_ref().unwrap(),
37                        c.provide_minio.as_ref().unwrap(),
38                        c.provide_aws_s3.as_ref().unwrap(),
39                        c.provide_moat.as_ref().unwrap(),
40                        HummockInMemoryStrategy::Disallowed,
41                        &mut cmd,
42                    )
43                    .is_ok()
44                    {
45                        writeln!(
46                            env,
47                            "RW_HUMMOCK_URL=\"{}\"",
48                            cmd.get_args().nth(1).unwrap().to_str().unwrap()
49                        )
50                        .unwrap();
51                    }
52                }
53
54                // RW_META_ADDR
55                {
56                    let meta_node = &c.provide_meta_node.as_ref().unwrap()[0];
57                    writeln!(
58                        env,
59                        "RW_META_ADDR=\"http://{}:{}\"",
60                        meta_node.address, meta_node.port
61                    )
62                    .unwrap();
63                }
64            }
65            ServiceConfig::Frontend(c) => {
66                let listen_address = &c.listen_address;
67                writeln!(
68                    env,
69                    "RISEDEV_RW_FRONTEND_LISTEN_ADDRESS=\"{listen_address}\"",
70                )
71                .unwrap();
72                let port = &c.port;
73                writeln!(env, "RISEDEV_RW_FRONTEND_PORT=\"{port}\"",).unwrap();
74            }
75            ServiceConfig::Kafka(c) => {
76                let brokers = format!("{}:{}", c.address, c.port);
77                writeln!(env, r#"RISEDEV_KAFKA_BOOTSTRAP_SERVERS="{brokers}""#,).unwrap();
78                writeln!(env, r#"RISEDEV_KAFKA_WITH_OPTIONS_COMMON="connector='kafka',properties.bootstrap.server='{brokers}'""#).unwrap();
79                writeln!(env, r#"RPK_BROKERS="{brokers}""#).unwrap();
80            }
81            ServiceConfig::SchemaRegistry(c) => {
82                let url = format!("http://{}:{}", c.address, c.port);
83                writeln!(env, r#"RISEDEV_SCHEMA_REGISTRY_URL="{url}""#,).unwrap();
84                writeln!(env, r#"RPK_REGISTRY_HOSTS="{url}""#).unwrap();
85            }
86            ServiceConfig::Pulsar(c) => {
87                // These 2 names are NOT defined by Pulsar, but by us.
88                // The `pulsar-admin` CLI uses a `PULSAR_CLIENT_CONF` file with `brokerServiceUrl` and `webServiceUrl`
89                // It may be used by our upcoming `PulsarCat` #21401
90                writeln!(
91                    env,
92                    r#"PULSAR_BROKER_URL="pulsar://{}:{}""#,
93                    c.address, c.broker_port
94                )
95                .unwrap();
96                writeln!(
97                    env,
98                    r#"PULSAR_HTTP_URL="http://{}:{}""#,
99                    c.address, c.http_port
100                )
101                .unwrap();
102            }
103            ServiceConfig::Redis(c) => {
104                let host = &c.address;
105                let port = &c.port;
106                let url = format!("redis://{host}:{port}/");
107                writeln!(env, r#"REDIS_HOST="{host}""#).unwrap();
108                writeln!(env, r#"REDIS_PORT="{port}""#).unwrap();
109                writeln!(env, r#"RISEDEV_REDIS_URL="{url}""#).unwrap();
110                if !c.user_managed {
111                    writeln!(env, r#"REDIS_CONTAINER="risedev-{}""#, c.id).unwrap();
112                }
113                writeln!(
114                    env,
115                    r#"RISEDEV_REDIS_WITH_OPTIONS_COMMON="connector='redis',redis.url='{url}'""#,
116                )
117                .unwrap();
118            }
119            ServiceConfig::MySql(c) if c.application != Application::Metastore => {
120                let host = &c.address;
121                let port = &c.port;
122                let user = &c.user;
123                let password = &c.password;
124                // These envs are used by `mysql` cli.
125                writeln!(env, r#"MYSQL_HOST="{host}""#,).unwrap();
126                writeln!(env, r#"MYSQL_TCP_PORT="{port}""#,).unwrap();
127                // Note: There's no env var for the username read by `mysql` cli. Here we set
128                // `RISEDEV_MYSQL_USER`, which will be read by `e2e_test/commands/mysql` when
129                // running `risedev slt`, as a wrapper of `mysql` cli.
130                writeln!(env, r#"RISEDEV_MYSQL_USER="{user}""#,).unwrap();
131                writeln!(env, r#"MYSQL_PWD="{password}""#,).unwrap();
132                // Note: user and password are not included in the common WITH options.
133                // It's expected to create another dedicated user for the source.
134                writeln!(env, r#"RISEDEV_MYSQL_WITH_OPTIONS_COMMON="connector='mysql-cdc',hostname='{host}',port='{port}'""#,).unwrap();
135            }
136            ServiceConfig::Pubsub(c) => {
137                let address = &c.address;
138                let port = &c.port;
139                writeln!(env, r#"PUBSUB_EMULATOR_HOST="{address}:{port}""#,).unwrap();
140                writeln!(env, r#"RISEDEV_PUBSUB_WITH_OPTIONS_COMMON="connector='google_pubsub',pubsub.emulator_host='{address}:{port}'""#,).unwrap();
141            }
142            ServiceConfig::Postgres(c) => {
143                let host = &c.address;
144                let port = &c.port;
145                let user = &c.user;
146                let password = &c.password;
147                let database = &c.database;
148                // These envs are used by `postgres` cli.
149                writeln!(env, r#"PGHOST="{host}""#,).unwrap();
150                writeln!(env, r#"PGPORT="{port}""#,).unwrap();
151                writeln!(env, r#"PGUSER="{user}""#,).unwrap();
152                writeln!(env, r#"PGPASSWORD="{password}""#,).unwrap();
153                writeln!(env, r#"PGDATABASE="{database}""#,).unwrap();
154                writeln!(
155                    env,
156                    r#"RISEDEV_POSTGRES_WITH_OPTIONS_COMMON="connector='postgres-cdc',hostname='{host}',port='{port}'""#,
157                )
158                .unwrap();
159            }
160            ServiceConfig::SqlServer(c) => {
161                let host = &c.address;
162                let port = &c.port;
163                let user = &c.user;
164                let password = &c.password;
165                let database = &c.database;
166                // These envs are used by `sqlcmd`.
167                writeln!(env, r#"SQLCMDSERVER="{host}""#,).unwrap();
168                writeln!(env, r#"SQLCMDPORT="{port}""#,).unwrap();
169                writeln!(env, r#"SQLCMDUSER="{user}""#,).unwrap();
170                writeln!(env, r#"SQLCMDPASSWORD="{password}""#,).unwrap();
171                writeln!(env, r#"SQLCMDDBNAME="{database}""#,).unwrap();
172                writeln!(
173                    env,
174                    r#"RISEDEV_SQLSERVER_WITH_OPTIONS_COMMON="connector='sqlserver-cdc',hostname='{host}',port='{port}',username='{user}',password='{password}',database.name='{database}'""#,
175                )
176                .unwrap();
177            }
178            ServiceConfig::MongoDb(c) => {
179                let host = &c.address;
180                let port = &c.port;
181                let url = format!("mongodb://{host}:{port}/?replicaSet=rs0");
182                writeln!(env, r#"MONGODB_HOST="{host}""#).unwrap();
183                writeln!(env, r#"MONGODB_PORT="{port}""#).unwrap();
184                writeln!(env, r#"MONGODB_URL="{url}""#).unwrap();
185                if !c.user_managed {
186                    writeln!(env, r#"MONGODB_CONTAINER="risedev-{}""#, c.id).unwrap();
187                }
188                writeln!(
189                    env,
190                    r#"RISEDEV_MONGODB_WITH_OPTIONS_COMMON="connector='mongodb',mongodb.url='{url}'""#,
191                )
192                .unwrap();
193            }
194            ServiceConfig::ElasticSearch(c) => {
195                let host = &c.address;
196                let port = &c.port;
197                let user = &c.user;
198                let password = &c.password;
199                let url = format!("http://{host}:{port}");
200                writeln!(env, r#"ELASTICSEARCH_HOST="{host}""#).unwrap();
201                writeln!(env, r#"ELASTICSEARCH_PORT="{port}""#).unwrap();
202                writeln!(env, r#"ELASTICSEARCH_USER="{user}""#).unwrap();
203                writeln!(env, r#"ELASTICSEARCH_PASSWORD="{password}""#).unwrap();
204                writeln!(env, r#"RISEDEV_ELASTICSEARCH_URL="{url}""#).unwrap();
205                writeln!(
206                    env,
207                    r#"RISEDEV_ELASTICSEARCH_WITH_OPTIONS_COMMON="connector='elasticsearch',url='{url}',username='{user}',password='{password}'""#,
208                )
209                .unwrap();
210                writeln!(env, r#"SEARCH_SINK_CONNECTOR="elasticsearch""#).unwrap();
211                writeln!(env, r#"SEARCH_SINK_CONNECTION_TYPE="elasticsearch""#).unwrap();
212                writeln!(env, r#"SEARCH_SINK_URL="{url}""#).unwrap();
213                writeln!(env, r#"SEARCH_SINK_USER="{user}""#).unwrap();
214                writeln!(env, r#"SEARCH_SINK_PASSWORD="{password}""#).unwrap();
215                writeln!(
216                    env,
217                    r#"SEARCH_SINK_WITH_OPTIONS_COMMON="connector='elasticsearch',url='{url}',username='{user}',password='{password}'""#,
218                )
219                .unwrap();
220                writeln!(
221                    env,
222                    r#"SEARCH_SINK_ROUTE_WITH_OPTIONS="connector='elasticsearch',connection=es_conn""#,
223                )
224                .unwrap();
225            }
226            ServiceConfig::OpenSearch(c) => {
227                let host = &c.address;
228                let port = &c.port;
229                let user = &c.user;
230                let password = &c.password;
231                let url = format!("http://{host}:{port}");
232                writeln!(env, r#"OPENSEARCH_HOST="{host}""#).unwrap();
233                writeln!(env, r#"OPENSEARCH_PORT="{port}""#).unwrap();
234                writeln!(env, r#"OPENSEARCH_USER="{user}""#).unwrap();
235                writeln!(env, r#"OPENSEARCH_PASSWORD="{password}""#).unwrap();
236                writeln!(env, r#"RISEDEV_OPENSEARCH_URL="{url}""#).unwrap();
237                writeln!(
238                    env,
239                    r#"RISEDEV_OPENSEARCH_WITH_OPTIONS_COMMON="connector='opensearch',url='{url}',username='{user}',password='{password}'""#,
240                )
241                .unwrap();
242                writeln!(env, r#"SEARCH_SINK_CONNECTOR="opensearch""#).unwrap();
243                writeln!(env, r#"SEARCH_SINK_CONNECTION_TYPE="elasticsearch""#).unwrap();
244                writeln!(env, r#"SEARCH_SINK_URL="{url}""#).unwrap();
245                writeln!(env, r#"SEARCH_SINK_USER="{user}""#).unwrap();
246                writeln!(env, r#"SEARCH_SINK_PASSWORD="{password}""#).unwrap();
247                writeln!(
248                    env,
249                    r#"SEARCH_SINK_WITH_OPTIONS_COMMON="connector='opensearch',url='{url}',username='{user}',password='{password}'""#,
250                )
251                .unwrap();
252                writeln!(
253                    env,
254                    r#"SEARCH_SINK_ROUTE_WITH_OPTIONS="connector='opensearch',url='{url}',username='{user}',password='{password}'""#,
255                )
256                .unwrap();
257            }
258            ServiceConfig::Nats(c) => {
259                let server_url = format!("nats://{}:{}", c.address, c.port);
260                writeln!(env, r#"NATS_SERVER_URL="{server_url}""#).unwrap();
261                writeln!(env, r#"RISEDEV_NATS_SERVER_URL="{server_url}""#).unwrap();
262                writeln!(
263                    env,
264                    r#"RISEDEV_NATS_WITH_OPTIONS_COMMON="connector='nats',server_url='{server_url}'""#,
265                )
266                .unwrap();
267            }
268            ServiceConfig::Mqtt(c) => {
269                let url = format!("tcp://{}:{}", c.address, c.port);
270                writeln!(env, r#"MQTT_URL="{url}""#).unwrap();
271                writeln!(env, r#"RISEDEV_MQTT_URL="{url}""#).unwrap();
272                writeln!(
273                    env,
274                    r#"RISEDEV_MQTT_WITH_OPTIONS_COMMON="connector='mqtt',url='{url}'""#,
275                )
276                .unwrap();
277            }
278            ServiceConfig::Minio(c) => {
279                let endpoint = format!("http://{}:{}", c.address, c.port);
280                writeln!(env, r#"RISEDEV_MINIO_ENDPOINT="{endpoint}""#).unwrap();
281                writeln!(env, r#"RISEDEV_MINIO_ACCESS_KEY="{0}""#, c.root_user).unwrap();
282                writeln!(env, r#"RISEDEV_MINIO_SECRET_KEY="{0}""#, c.root_password).unwrap();
283
284                writeln!(
285                    env,
286                    r#"RISEDEV_DELTALAKE_LOCATION="s3a://deltalake/deltalake-test""#
287                )
288                .unwrap();
289                writeln!(env, r#"RISEDEV_DELTALAKE_S3_ENDPOINT="{endpoint}""#).unwrap();
290                writeln!(env, r#"RISEDEV_DELTALAKE_S3_REGION="us-east-1""#).unwrap();
291                writeln!(env, r#"RISEDEV_DELTALAKE_S3_ACCESS_KEY="{0}""#, c.root_user).unwrap();
292                writeln!(
293                    env,
294                    r#"RISEDEV_DELTALAKE_S3_SECRET_KEY="{0}""#,
295                    c.root_password
296                )
297                .unwrap();
298            }
299            ServiceConfig::MetaNode(meta_node_config) => {
300                writeln!(
301                    env,
302                    r#"RISEDEV_RW_META_DASHBOARD_ADDR="http://{}:{}""#,
303                    meta_node_config.address, meta_node_config.dashboard_port
304                )
305                .unwrap();
306            }
307            ServiceConfig::Lakekeeper(c) => {
308                let base_url = format!("http://{}:{}", c.address, c.port);
309                let catalog_url = format!("{}/catalog", base_url);
310                writeln!(env, r#"RISEDEV_LAKEKEEPER_URL="{base_url}""#,).unwrap();
311                writeln!(env, r#"LAKEKEEPER_CATALOG_URL="{catalog_url}""#,).unwrap();
312                writeln!(env, r#"RISEDEV_LAKEKEEPER_WITH_OPTIONS_COMMON="connector='iceberg',catalog.type='rest',catalog.uri='{catalog_url}'""#,).unwrap();
313            }
314            ServiceConfig::Moto(c) => {
315                let endpoint = format!("http://{}:{}", c.address, c.port);
316                let glue_access_key = "my_access_id";
317                let glue_secret_key = "my_secret_key";
318                writeln!(env, r#"RISEDEV_MOTO_ENDPOINT="{endpoint}""#).unwrap();
319                writeln!(env, r#"RISEDEV_GLUE_ENDPOINT="{endpoint}""#).unwrap();
320                writeln!(env, r#"RW_TEST_GLUE_ENDPOINT="{endpoint}""#).unwrap();
321                writeln!(env, r#"RW_TEST_GLUE_ACCESS_KEY="{glue_access_key}""#).unwrap();
322                writeln!(env, r#"RW_TEST_GLUE_SECRET_KEY="{glue_secret_key}""#).unwrap();
323
324                if let Some(minio_configs) = &c.provide_minio
325                    && let Some(minio) = minio_configs.first()
326                {
327                    let s3_endpoint = format!("http://{}:{}", minio.address, minio.port);
328                    let warehouse_path = format!("s3://{}/iceberg", minio.hummock_bucket);
329                    writeln!(env, r#"RW_TEST_GLUE_S3_ENDPOINT="{s3_endpoint}""#).unwrap();
330                    writeln!(env, r#"RW_TEST_GLUE_S3_ACCESS_KEY="{0}""#, minio.root_user).unwrap();
331                    writeln!(
332                        env,
333                        r#"RW_TEST_GLUE_S3_SECRET_KEY="{0}""#,
334                        minio.root_password
335                    )
336                    .unwrap();
337                    writeln!(env, r#"RW_TEST_GLUE_WAREHOUSE_PATH="{warehouse_path}""#).unwrap();
338                    writeln!(env, r#"RISEDEV_GLUE_RUST_WITH_OPTIONS_COMMON="connector='iceberg',catalog.type='glue_rust',catalog.uri='{endpoint}',glue.endpoint='{endpoint}',glue.region='us-east-1',glue.access.key='{glue_access_key}',glue.secret.key='{glue_secret_key}',warehouse.path='{warehouse_path}',s3.endpoint='{s3_endpoint}',s3.region='us-east-1',s3.access.key='{0}',s3.secret.key='{1}',s3.path.style.access='true'""#, minio.root_user, minio.root_password).unwrap();
339                    writeln!(env, r#"RISEDEV_GLUE_JNI_WITH_OPTIONS_COMMON="connector='iceberg',catalog.type='glue',catalog.uri='{endpoint}',glue.endpoint='{endpoint}',glue.region='us-east-1',glue.access.key='{glue_access_key}',glue.secret.key='{glue_secret_key}',warehouse.path='{warehouse_path}',s3.endpoint='{s3_endpoint}',s3.region='us-east-1',s3.access.key='{0}',s3.secret.key='{1}',s3.path.style.access='true'""#, minio.root_user, minio.root_password).unwrap();
340                }
341            }
342            _ => {}
343        }
344    }
345    env
346}
risedev/risedev_env.rs

risedev/
risedev_env.rs
