From c23f5ff62dc8c46a3133266dffdff54f7c679c88 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 8 May 2024 17:04:59 +0800 Subject: [PATCH 01/10] initial commit Signed-off-by: Bugen Zhao --- expand.libsonnet | 32 ++++ risedev-profiles.libsonnet | 21 +++ risedev-template.libsonnet | 342 +++++++++++++++++++++++++++++++++++++ risedev.jsonnet | 4 + 4 files changed, 399 insertions(+) create mode 100644 expand.libsonnet create mode 100644 risedev-profiles.libsonnet create mode 100644 risedev-template.libsonnet create mode 100644 risedev.jsonnet diff --git a/expand.libsonnet b/expand.libsonnet new file mode 100644 index 0000000000000..8a009fd765d69 --- /dev/null +++ b/expand.libsonnet @@ -0,0 +1,32 @@ +local pruneProvide = function(step) + { + [name]: step[name] + for name in std.objectFields(step) + if !std.startsWith(name, 'provide') + }; + +local mapStep = function(step, steps) + { + [name]: if std.startsWith(name, 'provide') then + [ + pruneProvide(otherStep) + for otherStep in steps + if std.startsWith(otherStep.id, std.rstripChars(step[name], '*')) + ] + else step[name] + for name in std.objectFields(step) + }; + +local mapSteps = function(steps) + [ + mapStep(step, steps) + for step in steps + ]; + +local mapRoot = function(allProfiles) + { + [name]: allProfiles[name] { steps: mapSteps(allProfiles[name].steps) } + for name in std.objectFields(allProfiles) + }; + +mapRoot diff --git a/risedev-profiles.libsonnet b/risedev-profiles.libsonnet new file mode 100644 index 0000000000000..f45abd9ec64f4 --- /dev/null +++ b/risedev-profiles.libsonnet @@ -0,0 +1,21 @@ +local _ = import 'risedev-template.libsonnet'; + +{ + default: { + configFile: 'src/config/example.toml', + steps: [ + _.metaNode, + _.computeNode { port: 4588, userManaged: true }, + _.computeNode { port: 4599 }, + _.minio, + _.frontend, + ], + }, + + 'default-with-monitoring': $.default { + steps+: [ + _.prometheus, + _.grafana, + ], + }, +} diff --git a/risedev-template.libsonnet b/risedev-template.libsonnet new file mode 100644 index 0000000000000..2b71491014336 --- /dev/null +++ b/risedev-template.libsonnet @@ -0,0 +1,342 @@ +// The `use` field specified in the above `risedev` section will refer to the templates below. +{ + minio: { + // Advertise address of MinIO s3 endpoint + address: '127.0.0.1', + // Advertise port of MinIO s3 endpoint + port: 9301, + // Listen address of MinIO endpoint + listenAddress: self.address, + // Console address of MinIO s3 endpoint + consoleAddress: '127.0.0.1', + // Console port of MinIO s3 endpoint + consolePort: 9400, + // Root username (can be used to login to MinIO console) + rootUser: 'hummockadmin', + // Root password (can be used to login to MinIO console) + rootPassword: 'hummockadmin', + // Bucket name to store hummock information + hummockBucket: 'hummock001', + // Id of this instance + id: 'minio', + // Prometheus nodes used by this MinIO + providePrometheus: 'prometheus*', + // Max concurrent api requests. + // see: https://github.com/minio/minio/blob/master/docs/throttle/README.md. + // '0' means this env var will use the default of minio. + apiRequestsMax: 0, + // Deadline for api requests. + // Empty string means this env var will use the default of minio. + apiRequestsDeadline: '', + }, + + etcd: { + // Id of this instance + id: 'etcd-' + self.port, + // Advertise address of the single-node etcd. + address: '127.0.0.1', + // Listen port of the single-node etcd. + port: 2388, + // Listen address + listenAddress: self.address, + // Peer listen port of the single-node etcd. + peerPort: 2389, + // Prometheus exporter listen port + exporterPort: 2379, + // Whether to enable fsync (NEVER SET TO TRUE IN PRODUCTION ENVIRONMENT!) + unsafeNoFsync: false, + // Other etcd nodes + provideEtcd: 'etcd*', + }, + + sqlite: { + // Id of this instance + id: 'sqlite', + // File name of the sqlite database + file: 'metadata.db', + }, + + computeNode: { + // Compute-node advertise address + address: '127.0.0.1', + // Listen address + listenAddress: self.address, + // Compute-node listen port + port: 5688, + // Prometheus exporter listen port + exporterPort: 1222, + // Id of this instance + id: 'compute-node-' + self.port, + // Whether to enable async stack trace for this compute node, `off`, `on`, or `verbose`. + // Considering the performance, `verbose` mode only effect under `release` profile with `debug_assertions` off. + asyncStackTrace: 'verbose', + // If `enable-tiered-cache` is true, hummock will use data directory as file cache. + enableTieredCache: false, + // Minio instances used by this compute node + provideMinio: 'minio*', + // OpenDAL storage backend used by this compute node + provideOpendal: 'opendal*', + // AWS s3 bucket used by this compute node + provideAwsS3: 'aws-s3*', + // Meta-nodes used by this compute node + provideMetaNode: 'meta-node*', + // Tempo used by this compute node + provideTempo: 'tempo*', + // If `user-managed` is true, this service will be started by user with the above config + userManaged: false, + // Total available memory for the compute node in bytes + totalMemoryBytes: 8589934592, + // Parallelism of tasks per compute node + parallelism: 4, + role: 'both', + }, + + metaNode: { + // Meta-node advertise address + address: '127.0.0.1', + // Meta-node listen port + port: 5690, + // Listen address + listenAddress: self.address, + // Dashboard listen port + dashboardPort: 5691, + // Prometheus exporter listen port + exporterPort: 1250, + // Id of this instance + id: 'meta-node-' + self.port, + // If `user-managed` is true, this service will be started by user with the above config + userManaged: false, + // Etcd backend config + provideEtcdBackend: 'etcd*', + // Sqlite backend config + provideSqliteBackend: 'sqlite*', + // Prometheus nodes used by dashboard service + providePrometheus: 'prometheus*', + // Sanity check: should use shared storage if there're multiple compute nodes + provideComputeNode: 'compute-node*', + // Sanity check: should start at lease one compactor if using shared object store + provideCompactor: 'compactor*', + // Minio instances used by the cluster + provideMinio: 'minio*', + // OpenDAL storage backend used by the cluster + provideOpendal: 'opendal*', + // AWS s3 bucket used by the cluster + provideAwsS3: 'aws-s3*', + // Tempo used by this meta node + provideTempo: 'tempo*', + // Whether to enable in-memory pure KV state backend + enableInMemoryKvStateBackend: false, + }, + + prometheus: { + // Advertise address of Prometheus + address: '127.0.0.1', + // Listen port of Prometheus + port: 9500, + // Listen address + listenAddress: self.address, + // Id of this instance + id: 'prometheus', + // If `remote_write` is true, this Prometheus instance will push metrics to remote instance + remoteWrite: false, + // AWS region of remote write + remoteWriteRegion: '', + // Remote write url of this instance + remoteWriteUrl: '', + // Compute-nodes used by this Prometheus instance + provideComputeNode: 'compute-node*', + // Meta-nodes used by this Prometheus instance + provideMetaNode: 'meta-node*', + // Minio instances used by this Prometheus instance + provideMinio: 'minio*', + // Compactors used by this Prometheus instance + provideCompactor: 'compactor*', + // Etcd used by this Prometheus instance + provideEtcd: 'etcd*', + // Redpanda used by this Prometheus instance + provideRedpanda: 'redpanda*', + // Frontend used by this Prometheus instance + provideFrontend: 'frontend*', + // How frequently Prometheus scrape targets (collect metrics) + scrapeInterval: '15s', + }, + + frontend: { + // Advertise address of frontend + address: '127.0.0.1', + // Listen port of frontend + port: 4566, + // Listen address + listenAddress: self.address, + // Prometheus exporter listen port + exporterPort: 2222, + // Health check listen port + healthCheckPort: 6786, + // Id of this instance + id: 'frontend-' + self.port, + // Meta-nodes used by this frontend instance + provideMetaNode: 'meta-node*', + // Tempo used by this frontend instance + provideTempo: 'tempo*', + // If `user-managed` is true, this service will be started by user with the above config + userManaged: false, + }, + + compactor: { + // Compactor advertise address + address: '127.0.0.1', + // Compactor listen port + port: 6660, + // Listen address + listenAddress: self.address, + // Prometheus exporter listen port + exporterPort: 1260, + // Id of this instance + id: 'compactor-' + self.port, + // Minio instances used by this compactor + provideMinio: 'minio*', + // Meta-nodes used by this compactor + provideMetaNode: 'meta-node*', + // Tempo used by this compator + provideTempo: 'tempo*', + // If `user-managed` is true, this service will be started by user with the above config + userManaged: false, + }, + + grafana: { + // Listen address of Grafana + listenAddress: self.address, + // Advertise address of Grafana + address: '127.0.0.1', + // Listen port of Grafana + port: 3001, + // Id of this instance + id: 'grafana', + // Prometheus used by this Grafana instance + providePrometheus: 'prometheus*', + // Tempo used by this Grafana instance + provideTempo: 'tempo*', + }, + + tempo: { + // Id of this instance + id: 'tempo', + // Listen address of HTTP server and OTLP gRPC collector + listenAddress: '127.0.0.1', + // Advertise address of Tempo + address: '127.0.0.1', + // HTTP server listen port + port: 3200, + // gRPC listen port of the OTLP collector + otlpPort: 4317, + maxBytesPerTrace: 5000000, + }, + + opendal: { + id: 'opendal', + engine: 'hdfs', + namenode: '127.0.0.1:9000', + bucket: 'risingwave-test', + }, + + // awsS3 is a placeholder service to provide configurations + awsS3: { + // Id to be picked-up by services + id: 'aws-s3', + // The bucket to be used for AWS S3 + bucket: 'test-bucket', + // access key, secret key and region should be set in aws config (either by env var or .aws/config) + }, + + // Apache Kafka service + kafka: { + // Id to be picked-up by services + id: 'kafka-' + self.port, + // Advertise address of Kafka + address: '127.0.0.1', + // Listen port of Kafka + port: 29092, + // Listen address + listenAddress: self.address, + // ZooKeeper used by this Kafka instance + provideZookeeper: 'zookeeper*', + // If set to true, data will be persisted at data/{id}. + persistData: true, + // Kafka broker id. If there are multiple instances of Kafka, we will need to set. + brokerId: 0, + userManaged: false, + }, + + // Google pubsub emulator service + pubsub: { + id: 'pubsub-' + self.port, + address: '127.0.0.1', + port: 5980, + persistData: true, + }, + + // Apache ZooKeeper service + zookeeper: { + // Id to be picked-up by services + id: 'zookeeper-' + self.port, + // Advertise address of ZooKeeper + address: '127.0.0.1', + // Listen address + listenAddress: self.address, + // Listen port of ZooKeeper + port: 2181, + // If set to true, data will be persisted at data/{id}. + persistData: true, + }, + + // Only supported in RiseDev compose + redpanda: { + // Id to be picked-up by services + id: 'redpanda', + // Port used inside docker-compose cluster (e.g. create MV) + internalPort: 29092, + // Port used on host (e.g. import data, connecting using kafkacat) + outsidePort: 9092, + // Connect address + address: self.id, + // Number of CPUs to use + cpus: 8, + // Memory limit for Redpanda + memory: '16G', + }, + + // redis service + redis: { + // Id to be picked-up by services + id: 'redis', + // listen port of redis + port: 6379, + // address of redis + address: '127.0.0.1', + }, + + // MySQL service backed by docker. + mysql: { + // Id to be picked-up by services + id: 'mysql-' + self.port, + // address of mysql + address: '127.0.0.1', + // listen port of mysql + port: 8306, + // Note: + // - This will be used to initialize the MySQL instance if it's fresh. + // - In user-managed mode, these configs are not validated by risedev. + // They are passed as-is to risedev-env default user for MySQL operations. + // - This is not used in RISEDEV_MYSQL_WITH_OPTIONS_COMMON. + user: 'root', + password: '', + database: 'risedev', + // The docker image. Can be overridden to use a different version. + image: 'mysql:8', + // If set to true, data will be persisted at data/{id}. + persistData: true, + // If `user-managed` is true, user is responsible for starting the service + // to serve at the above address and port in any way they see fit. + userManaged: false, + }, +} diff --git a/risedev.jsonnet b/risedev.jsonnet new file mode 100644 index 0000000000000..76291e87074f6 --- /dev/null +++ b/risedev.jsonnet @@ -0,0 +1,4 @@ +local expand = import 'expand.libsonnet'; +local profiles = import 'risedev-profiles.libsonnet'; + +expand(profiles)[std.extVar('profile')] From 59b16207c8b3d89c39eeecaba004073bfff5350e Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 16:06:52 +0800 Subject: [PATCH 02/10] refine more Signed-off-by: Bugen Zhao --- Makefile.toml | 3 + expand.libsonnet => risedev-expand.libsonnet | 0 risedev-profiles.libsonnet | 7 +- risedev-template.libsonnet | 7 +- risedev.jsonnet | 10 ++- src/risedevtool/src/bin/risedev-compose.rs | 2 +- src/risedevtool/src/bin/risedev-dev.rs | 51 +++++++++++---- src/risedevtool/src/config.rs | 2 +- src/risedevtool/src/risedev_env.rs | 2 +- src/risedevtool/src/service_config.rs | 68 ++++++++++---------- 10 files changed, 93 insertions(+), 59 deletions(-) rename expand.libsonnet => risedev-expand.libsonnet (100%) diff --git a/Makefile.toml b/Makefile.toml index 910642dc8beae..6802964be2068 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -668,6 +668,9 @@ alias = "dev" category = "RiseDev - Start/Stop" dependencies = ["pre-start-dev"] description = "🌟 Start a full RisingWave dev cluster using risedev-dev" +install_crate = { min_version = "0.1.1", crate_name = "rsjsonnet", binary = "rsjsonnet", test_arg = [ + "--help", +] } script = ''' #!/usr/bin/env bash diff --git a/expand.libsonnet b/risedev-expand.libsonnet similarity index 100% rename from expand.libsonnet rename to risedev-expand.libsonnet diff --git a/risedev-profiles.libsonnet b/risedev-profiles.libsonnet index f45abd9ec64f4..9c972dd73fd85 100644 --- a/risedev-profiles.libsonnet +++ b/risedev-profiles.libsonnet @@ -5,9 +5,10 @@ local _ = import 'risedev-template.libsonnet'; configFile: 'src/config/example.toml', steps: [ _.metaNode, - _.computeNode { port: 4588, userManaged: true }, - _.computeNode { port: 4599 }, - _.minio, + _.computeNode, + // _.computeNode { port: 4588, userManaged: true }, + // _.computeNode { port: 4599 }, + // _.minio, _.frontend, ], }, diff --git a/risedev-template.libsonnet b/risedev-template.libsonnet index 2b71491014336..2049e88fbf6ca 100644 --- a/risedev-template.libsonnet +++ b/risedev-template.libsonnet @@ -1,5 +1,5 @@ // The `use` field specified in the above `risedev` section will refer to the templates below. -{ +local templates = { minio: { // Advertise address of MinIO s3 endpoint address: '127.0.0.1', @@ -339,4 +339,9 @@ // to serve at the above address and port in any way they see fit. userManaged: false, }, +}; + +{ + [name]: templates[name] { use: name } + for name in std.objectFields(templates) } diff --git a/risedev.jsonnet b/risedev.jsonnet index 76291e87074f6..b288b055c399d 100644 --- a/risedev.jsonnet +++ b/risedev.jsonnet @@ -1,4 +1,8 @@ -local expand = import 'expand.libsonnet'; -local profiles = import 'risedev-profiles.libsonnet'; +local expand = import 'risedev-expand.libsonnet'; +local allProfiles = import 'risedev-profiles.libsonnet'; +local profile = std.extVar('profile'); -expand(profiles)[std.extVar('profile')] +if profile in allProfiles then + expand(allProfiles)[profile] { profile: profile } +else + error 'unknown profile ' + profile diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index c3a7d079e4c4e..46a0314846c92 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -222,7 +222,7 @@ fn main() -> Result<()> { volumes.insert(c.id.clone(), ComposeVolume::default()); (c.address.clone(), c.compose(&compose_config)?) } - ServiceConfig::Redis(_) | ServiceConfig::MySql(_) => { + ServiceConfig::Redis(_) | ServiceConfig::Mysql(_) => { return Err(anyhow!("not supported")) } }; diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index 3c5ad8aa082e0..c2e5247d94662 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(iterator_try_collect)] + use std::env; use std::fmt::Write; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use console::style; use fs_err::OpenOptions; use indicatif::ProgressBar; @@ -30,6 +32,7 @@ use risedev::{ PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, TempoService, ZooKeeperService, RISEDEV_NAME, }; +use serde::{Deserialize, Serialize}; use tempfile::tempdir; use thiserror_ext::AsReport; use yaml_rust::YamlEmitter; @@ -316,7 +319,7 @@ fn task_main( ctx.pb .set_message(format!("redis {}:{}", c.address, c.port)); } - ServiceConfig::MySql(c) => { + ServiceConfig::Mysql(c) => { let mut ctx = ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone()); MySqlService::new(c.clone()).execute(&mut ctx)?; @@ -336,6 +339,13 @@ fn task_main( Ok((stat, log_buffer)) } +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Input { + profile: String, + config_file: Option, + steps: Vec, +} + fn main() -> Result<()> { // Intentionally disable backtrace to provide more compact error message for `risedev dev`. // Backtraces for RisingWave components are enabled in `Task::execute`. @@ -347,24 +357,37 @@ fn main() -> Result<()> { .nth(1) .unwrap_or_else(|| "default".to_string()); - let (config_path, risedev_config) = ConfigExpander::expand(".", &task_name)?; + let json = std::process::Command::new("rsjsonnet") + .arg("risedev.jsonnet") + .arg("-V") + .arg(format!("profile={task_name}")) + .output() + .context("failed to evaluate risedev.jsonnet")?; + + if !json.status.success() { + bail!( + "failed to evaluate RiseDev profile configuration:\n{}", + String::from_utf8_lossy(&json.stderr) + ); + } + let input = json.stdout; + + fs_err::write( + Path::new(&env::var("PREFIX_CONFIG")?).join("risedev-profile-expanded.json"), + &input, + )?; + + let Input { + profile: task_name, + config_file: config_path, + steps: services, + }: Input = serde_json::from_slice(&input).context("failed to parse input")?; if let Some(config_path) = &config_path { let target = Path::new(&env::var("PREFIX_CONFIG")?).join("risingwave.toml"); fs_err::copy(config_path, target).context("config file not found")?; } - { - let mut out_str = String::new(); - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&risedev_config)?; - fs_err::write( - Path::new(&env::var("PREFIX_CONFIG")?).join("risedev-expanded.yml"), - &out_str, - )?; - } - let services = ConfigExpander::deserialize(&risedev_config)?; - let mut manager = ProgressManager::new(); // Always create a progress before calling `task_main`. Otherwise the progress bar won't be // shown. diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs index 45d90daa0c872..cfbae23fc9371 100644 --- a/src/risedevtool/src/config.rs +++ b/src/risedevtool/src/config.rs @@ -174,7 +174,7 @@ impl ConfigExpander { "redis" => ServiceConfig::Redis(serde_yaml::from_str(&out_str)?), "zookeeper" => ServiceConfig::ZooKeeper(serde_yaml::from_str(&out_str)?), "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?), - "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?), + "mysql" => ServiceConfig::Mysql(serde_yaml::from_str(&out_str)?), other => return Err(anyhow!("unsupported use type: {}", other)), }; Ok(result) diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs index 24f18895434e9..e6e9b5ccac163 100644 --- a/src/risedevtool/src/risedev_env.rs +++ b/src/risedevtool/src/risedev_env.rs @@ -77,7 +77,7 @@ pub fn generate_risedev_env(services: &Vec) -> String { writeln!(env, r#"RISEDEV_KAFKA_WITH_OPTIONS_COMMON="connector='kafka',properties.bootstrap.server='{brokers}'""#).unwrap(); writeln!(env, r#"RPK_BROKERS="{brokers}""#).unwrap(); } - ServiceConfig::MySql(c) => { + ServiceConfig::Mysql(c) => { let host = &c.address; let port = &c.port; let user = &c.user; diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 27231552dfc2d..2cce28e7f4e09 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct ComputeNodeConfig { #[serde(rename = "use")] @@ -23,7 +23,7 @@ pub struct ComputeNodeConfig { pub id: String, pub address: String, - #[serde(with = "string")] + pub port: u16, pub listen_address: String, pub exporter_port: u16, @@ -44,7 +44,7 @@ pub struct ComputeNodeConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct MetaNodeConfig { #[serde(rename = "use")] @@ -52,7 +52,6 @@ pub struct MetaNodeConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, pub dashboard_port: u16, @@ -76,7 +75,7 @@ pub struct MetaNodeConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct FrontendConfig { #[serde(rename = "use")] @@ -84,7 +83,6 @@ pub struct FrontendConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, pub exporter_port: u16, @@ -97,7 +95,7 @@ pub struct FrontendConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct CompactorConfig { #[serde(rename = "use")] @@ -105,7 +103,6 @@ pub struct CompactorConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, pub exporter_port: u16, @@ -120,7 +117,7 @@ pub struct CompactorConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct MinioConfig { #[serde(rename = "use")] @@ -128,12 +125,10 @@ pub struct MinioConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, pub console_address: String, - #[serde(with = "string")] pub console_port: u16, pub root_user: String, @@ -148,7 +143,7 @@ pub struct MinioConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct EtcdConfig { #[serde(rename = "use")] @@ -157,7 +152,6 @@ pub struct EtcdConfig { // TODO: only one node etcd is supported. pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, @@ -170,7 +164,7 @@ pub struct EtcdConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct SqliteConfig { #[serde(rename = "use")] @@ -181,7 +175,7 @@ pub struct SqliteConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct PrometheusConfig { #[serde(rename = "use")] @@ -189,7 +183,6 @@ pub struct PrometheusConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, @@ -209,7 +202,7 @@ pub struct PrometheusConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct GrafanaConfig { #[serde(rename = "use")] @@ -224,7 +217,7 @@ pub struct GrafanaConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct TempoConfig { #[serde(rename = "use")] @@ -239,7 +232,7 @@ pub struct TempoConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct AwsS3Config { #[serde(rename = "use")] @@ -249,7 +242,7 @@ pub struct AwsS3Config { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct OpendalConfig { #[serde(rename = "use")] @@ -262,7 +255,7 @@ pub struct OpendalConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct KafkaConfig { #[serde(rename = "use")] @@ -270,7 +263,6 @@ pub struct KafkaConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, @@ -281,13 +273,12 @@ pub struct KafkaConfig { pub user_managed: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct PubsubConfig { #[serde(rename = "use")] phantom_use: Option, pub id: String, - #[serde(with = "string")] pub port: u16, pub address: String, @@ -295,7 +286,7 @@ pub struct PubsubConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct ZooKeeperConfig { #[serde(rename = "use")] @@ -303,7 +294,6 @@ pub struct ZooKeeperConfig { pub id: String, pub address: String, - #[serde(with = "string")] pub port: u16, pub listen_address: String, @@ -311,7 +301,7 @@ pub struct ZooKeeperConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct RedPandaConfig { #[serde(rename = "use")] @@ -325,19 +315,18 @@ pub struct RedPandaConfig { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct RedisConfig { #[serde(rename = "use")] phantom_use: Option, pub id: String, - pub port: u16, pub address: String, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] pub struct MySqlConfig { #[serde(rename = "use")] @@ -357,7 +346,8 @@ pub struct MySqlConfig { } /// All service configuration -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "use", rename_all = "camelCase")] pub enum ServiceConfig { ComputeNode(ComputeNodeConfig), MetaNode(MetaNodeConfig), @@ -376,9 +366,17 @@ pub enum ServiceConfig { Redis(RedisConfig), ZooKeeper(ZooKeeperConfig), RedPanda(RedPandaConfig), - MySql(MySqlConfig), + Mysql(MySqlConfig), } +// impl TryFrom for ServiceConfig { +// type Error = anyhow::Error; + +// fn try_from(value: serde_json::Value) -> Result { +// todo!() +// } +// } + impl ServiceConfig { pub fn id(&self) -> &str { match self { @@ -399,7 +397,7 @@ impl ServiceConfig { Self::Redis(c) => &c.id, Self::RedPanda(c) => &c.id, Self::Opendal(c) => &c.id, - Self::MySql(c) => &c.id, + Self::Mysql(c) => &c.id, } } @@ -422,7 +420,7 @@ impl ServiceConfig { Self::Redis(c) => Some(c.port), Self::RedPanda(_c) => None, Self::Opendal(_) => None, - Self::MySql(c) => Some(c.port), + Self::Mysql(c) => Some(c.port), } } @@ -445,7 +443,7 @@ impl ServiceConfig { Self::Redis(_c) => false, Self::RedPanda(_c) => false, Self::Opendal(_c) => false, - Self::MySql(c) => c.user_managed, + Self::Mysql(c) => c.user_managed, } } } From 83fcdaa32c656ac600529c2a381d67f74a8be6a3 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 16:40:10 +0800 Subject: [PATCH 03/10] compat with old risedev Signed-off-by: Bugen Zhao --- risedev-profiles-compat.libsonnet | 27 ++ risedev.jsonnet | 6 +- risedev.yml | 589 +----------------------------- 3 files changed, 34 insertions(+), 588 deletions(-) create mode 100644 risedev-profiles-compat.libsonnet diff --git a/risedev-profiles-compat.libsonnet b/risedev-profiles-compat.libsonnet new file mode 100644 index 0000000000000..8abbec644abeb --- /dev/null +++ b/risedev-profiles-compat.libsonnet @@ -0,0 +1,27 @@ +local yaml = importstr 'risedev.yml'; +local profiles = std.parseYaml(yaml).profile; + +local _ = import 'risedev-template.libsonnet'; + +local kebabToCamel = function(s) + std.join('', std.mapWithIndex( + function(i, x) if i == 0 then x else std.asciiUpper(x[0]) + x[1:] + , std.split(s, '-') + )); + +local mapStep = function(step) + _[kebabToCamel(step.use)] + { + [kebabToCamel(name)]: step[name] + for name in std.objectFields(step) + if name != 'use' + } +; + +local mapProfile = function(name, profile) + (if 'config-path' in profile then { configPath: profile['config-path'] } else {}) + { + steps: std.map(mapStep, profile.steps), + }; + +std.mapWithKey(mapProfile, profiles) diff --git a/risedev.jsonnet b/risedev.jsonnet index b288b055c399d..d4d50308acb71 100644 --- a/risedev.jsonnet +++ b/risedev.jsonnet @@ -1,5 +1,9 @@ local expand = import 'risedev-expand.libsonnet'; -local allProfiles = import 'risedev-profiles.libsonnet'; + +local compatProfiles = import 'risedev-profiles-compat.libsonnet'; +local profiles = import 'risedev-profiles.libsonnet'; +local allProfiles = profiles + compatProfiles; + local profile = std.extVar('profile'); if profile in allProfiles then diff --git a/risedev.yml b/risedev.yml index ec057c5bf2c4c..4981ebe714553 100644 --- a/risedev.yml +++ b/risedev.yml @@ -416,140 +416,6 @@ profile: # port: 9092 # persist-data: true - ###################################### - ### Configurations used in Compose ### - ###################################### - - compose: - steps: - - use: minio - id: minio-0 - address: ${id} - listen-address: "0.0.0.0" - console-address: "0.0.0.0" - - - use: meta-node - # Id must starts with `meta-node`, therefore to be picked up by other - # components. - id: meta-node-0 - - # Advertise address can be `id`, so as to use docker's DNS. If running - # in host network mode, we should use IP directly in this field. - address: ${id} - - listen-address: "0.0.0.0" - - - use: compute-node - id: compute-node-0 - listen-address: "0.0.0.0" - address: ${id} - - - use: frontend - id: frontend-node-0 - listen-address: "0.0.0.0" - address: ${id} - - - use: compactor - id: compactor-0 - listen-address: "0.0.0.0" - address: ${id} - - - use: redpanda - - - use: prometheus - id: prometheus-0 - listen-address: "0.0.0.0" - address: ${id} - - - use: grafana - listen-address: "0.0.0.0" - address: ${id} - id: grafana-0 - - - use: tempo - listen-address: "0.0.0.0" - address: ${id} - id: tempo-0 - - - use: etcd - listen-address: "0.0.0.0" - address: ${id} - id: etcd-0 - - # special config for deployment, see related PR for more information - compose-3node-deploy: - steps: - # - use: minio - # id: minio-0 - # address: ${dns-host:rw-source-0} - # listen-address: "0.0.0.0" - # console-address: "0.0.0.0" - - - use: aws-s3 - bucket: ${terraform:s3-bucket} - - # Not enabled by default as all previous benchmarks are not done with etcd. - # Also we currently only support node-level docker volume tear down. - # - use: etcd - # listen-address: "0.0.0.0" - # address: ${dns-host:rw-meta-0} - # id: etcd-0 - - - use: meta-node - # Id must starts with `meta-node`, therefore to be picked up by other - # components. - id: meta-node-0 - - # Advertise address can be `id`, so as to use docker's DNS. If running - # in host network mode, we should use IP directly in this field. - address: ${dns-host:rw-meta-0} - listen-address: "0.0.0.0" - - - use: compute-node - id: compute-node-0 - listen-address: "0.0.0.0" - address: ${dns-host:rw-compute-0} - async-stack-trace: verbose - enable-tiered-cache: true - - - use: compute-node - id: compute-node-1 - listen-address: "0.0.0.0" - address: ${dns-host:rw-compute-1} - async-stack-trace: verbose - enable-tiered-cache: true - - - use: compute-node - id: compute-node-2 - listen-address: "0.0.0.0" - address: ${dns-host:rw-compute-2} - async-stack-trace: verbose - enable-tiered-cache: true - - - use: frontend - id: frontend-node-0 - listen-address: "0.0.0.0" - address: ${dns-host:rw-meta-0} - - - use: compactor - id: compactor-0 - listen-address: "0.0.0.0" - address: ${dns-host:rw-source-0} - compaction-worker-threads-number: 15 - - - use: redpanda - address: ${dns-host:rw-source-0} - - - use: prometheus - id: prometheus-0 - listen-address: "0.0.0.0" - address: ${dns-host:rw-meta-0} - - - use: grafana - listen-address: "0.0.0.0" - address: ${dns-host:rw-meta-0} - id: grafana-0 - ################################# ### Configurations used on CI ### ################################# @@ -1022,457 +888,6 @@ profile: - use: prometheus - use: grafana -compose: - risingwave: "ghcr.io/risingwavelabs/risingwave:latest" - prometheus: "prom/prometheus:latest" - minio: "quay.io/minio/minio:latest" - redpanda: "docker.vectorized.io/vectorized/redpanda:latest" - grafana: "grafana/grafana-oss:latest" - etcd: "quay.io/coreos/etcd:latest" - tempo: "grafana/tempo:latest" - # The `use` field specified in the above `risedev` section will refer to the templates below. -template: - minio: - # Advertise address of MinIO s3 endpoint - address: "127.0.0.1" - - # Advertise port of MinIO s3 endpoint - port: 9301 - - # Listen address of MinIO endpoint - listen-address: ${address} - - # Console address of MinIO s3 endpoint - console-address: "127.0.0.1" - - # Console port of MinIO s3 endpoint - console-port: 9400 - - # Root username (can be used to login to MinIO console) - root-user: hummockadmin - - # Root password (can be used to login to MinIO console) - root-password: hummockadmin - - # Bucket name to store hummock information - hummock-bucket: hummock001 - - # Id of this instance - id: minio - - # Prometheus nodes used by this MinIO - provide-prometheus: "prometheus*" - - # Max concurrent api requests. - # see: https://github.com/minio/minio/blob/master/docs/throttle/README.md. - # '0' means this env var will use the default of minio. - api-requests-max: 0 - - # Deadline for api requests. - # Empty string means this env var will use the default of minio. - api-requests-deadline: "" - - etcd: - # Id of this instance - id: etcd-${port} - - # Advertise address of the single-node etcd. - address: "127.0.0.1" - - # Listen port of the single-node etcd. - port: 2388 - - # Listen address - listen-address: ${address} - - # Peer listen port of the single-node etcd. - peer-port: 2389 - - # Prometheus exporter listen port - exporter-port: 2379 - - # Whether to enable fsync (NEVER SET TO TRUE IN PRODUCTION ENVIRONMENT!) - unsafe-no-fsync: false - - # Other etcd nodes - provide-etcd: "etcd*" - - sqlite: - # Id of this instance - id: sqlite - - # File name of the sqlite database - file: metadata.db - - compute-node: - # Compute-node advertise address - address: "127.0.0.1" - - # Listen address - listen-address: ${address} - - # Compute-node listen port - port: 5688 - - # Prometheus exporter listen port - exporter-port: 1222 - - # Id of this instance - id: compute-node-${port} - - # Whether to enable async stack trace for this compute node, `off`, `on`, or `verbose`. - # Considering the performance, `verbose` mode only effect under `release` profile with `debug_assertions` off. - async-stack-trace: verbose - - # If `enable-tiered-cache` is true, hummock will use data directory as file cache. - enable-tiered-cache: false - - # Minio instances used by this compute node - provide-minio: "minio*" - - # OpenDAL storage backend used by this compute node - provide-opendal: "opendal*" - - # AWS s3 bucket used by this compute node - provide-aws-s3: "aws-s3*" - - # Meta-nodes used by this compute node - provide-meta-node: "meta-node*" - - # Tempo used by this compute node - provide-tempo: "tempo*" - - # If `user-managed` is true, this service will be started by user with the above config - user-managed: false - - # Total available memory for the compute node in bytes - total-memory-bytes: 8589934592 - - # Parallelism of tasks per compute node - parallelism: 4 - - role: both - - meta-node: - # Meta-node advertise address - address: "127.0.0.1" - - # Meta-node listen port - port: 5690 - - # Listen address - listen-address: ${address} - - # Dashboard listen port - dashboard-port: 5691 - - # Prometheus exporter listen port - exporter-port: 1250 - - # Id of this instance - id: meta-node-${port} - - # If `user-managed` is true, this service will be started by user with the above config - user-managed: false - - # Etcd backend config - provide-etcd-backend: "etcd*" - - # Sqlite backend config - provide-sqlite-backend: "sqlite*" - - # Prometheus nodes used by dashboard service - provide-prometheus: "prometheus*" - - # Sanity check: should use shared storage if there're multiple compute nodes - provide-compute-node: "compute-node*" - - # Sanity check: should start at lease one compactor if using shared object store - provide-compactor: "compactor*" - - # Minio instances used by the cluster - provide-minio: "minio*" - - # OpenDAL storage backend used by the cluster - provide-opendal: "opendal*" - - # AWS s3 bucket used by the cluster - provide-aws-s3: "aws-s3*" - - # Tempo used by this meta node - provide-tempo: "tempo*" - - # Whether to enable in-memory pure KV state backend - enable-in-memory-kv-state-backend: false - - prometheus: - # Advertise address of Prometheus - address: "127.0.0.1" - - # Listen port of Prometheus - port: 9500 - - # Listen address - listen-address: ${address} - - # Id of this instance - id: prometheus - - # If `remote_write` is true, this Prometheus instance will push metrics to remote instance - remote-write: false - - # AWS region of remote write - remote-write-region: "" - - # Remote write url of this instance - remote-write-url: "" - - # Compute-nodes used by this Prometheus instance - provide-compute-node: "compute-node*" - - # Meta-nodes used by this Prometheus instance - provide-meta-node: "meta-node*" - - # Minio instances used by this Prometheus instance - provide-minio: "minio*" - - # Compactors used by this Prometheus instance - provide-compactor: "compactor*" - - # Etcd used by this Prometheus instance - provide-etcd: "etcd*" - - # Redpanda used by this Prometheus instance - provide-redpanda: "redpanda*" - - # Frontend used by this Prometheus instance - provide-frontend: "frontend*" - - # How frequently Prometheus scrape targets (collect metrics) - scrape-interval: 15s - - frontend: - # Advertise address of frontend - address: "127.0.0.1" - - # Listen port of frontend - port: 4566 - - # Listen address - listen-address: ${address} - - # Prometheus exporter listen port - exporter-port: 2222 - - # Health check listen port - health-check-port: 6786 - - # Id of this instance - id: frontend-${port} - - # Meta-nodes used by this frontend instance - provide-meta-node: "meta-node*" - - # Tempo used by this frontend instance - provide-tempo: "tempo*" - - # If `user-managed` is true, this service will be started by user with the above config - user-managed: false - - compactor: - # Compactor advertise address - address: "127.0.0.1" - - # Compactor listen port - port: 6660 - - # Listen address - listen-address: ${address} - - # Prometheus exporter listen port - exporter-port: 1260 - - # Id of this instance - id: compactor-${port} - - # Minio instances used by this compactor - provide-minio: "minio*" - - # Meta-nodes used by this compactor - provide-meta-node: "meta-node*" - - # Tempo used by this compator - provide-tempo: "tempo*" - - # If `user-managed` is true, this service will be started by user with the above config - user-managed: false - - grafana: - # Listen address of Grafana - listen-address: ${address} - - # Advertise address of Grafana - address: "127.0.0.1" - - # Listen port of Grafana - port: 3001 - - # Id of this instance - id: grafana - - # Prometheus used by this Grafana instance - provide-prometheus: "prometheus*" - - # Tempo used by this Grafana instance - provide-tempo: "tempo*" - - tempo: - # Id of this instance - id: tempo - - # Listen address of HTTP server and OTLP gRPC collector - listen-address: "127.0.0.1" - - # Advertise address of Tempo - address: "127.0.0.1" - - # HTTP server listen port - port: 3200 - - # gRPC listen port of the OTLP collector - otlp-port: 4317 - - max-bytes-per-trace: 5000000 - - opendal: - id: opendal - - engine: hdfs - - namenode: 127.0.0.1:9000 - - bucket: risingwave-test - - # aws-s3 is a placeholder service to provide configurations - aws-s3: - # Id to be picked-up by services - id: aws-s3 - - # The bucket to be used for AWS S3 - bucket: test-bucket - - # access key, secret key and region should be set in aws config (either by env var or .aws/config) - - # Apache Kafka service - kafka: - # Id to be picked-up by services - id: kafka-${port} - - # Advertise address of Kafka - address: "127.0.0.1" - - # Listen port of Kafka - port: 29092 - - # Listen address - listen-address: ${address} - - # ZooKeeper used by this Kafka instance - provide-zookeeper: "zookeeper*" - - # If set to true, data will be persisted at data/{id}. - persist-data: true - - # Kafka broker id. If there are multiple instances of Kafka, we will need to set. - broker-id: 0 - - user-managed: false - - # Google pubsub emulator service - pubsub: - id: pubsub-${port} - - address: "127.0.0.1" - - port: 5980 - - persist-data: true - - # Apache ZooKeeper service - zookeeper: - # Id to be picked-up by services - id: zookeeper-${port} - - # Advertise address of ZooKeeper - address: "127.0.0.1" - - # Listen address - listen-address: ${address} - - # Listen port of ZooKeeper - port: 2181 - - # If set to true, data will be persisted at data/{id}. - persist-data: true - - # Only supported in RiseDev compose - redpanda: - # Id to be picked-up by services - id: redpanda - - # Port used inside docker-compose cluster (e.g. create MV) - internal-port: 29092 - - # Port used on host (e.g. import data, connecting using kafkacat) - outside-port: 9092 - - # Connect address - address: ${id} - - # Number of CPUs to use - cpus: 8 - - # Memory limit for Redpanda - memory: 16G - - # redis service - redis: - # Id to be picked-up by services - id: redis - - # listen port of redis - port: 6379 - - # address of redis - address: "127.0.0.1" - - # MySQL service backed by docker. - mysql: - # Id to be picked-up by services - id: mysql-${port} - - # address of mysql - address: "127.0.0.1" - - # listen port of mysql - port: 8306 - - # Note: - # - This will be used to initialize the MySQL instance if it's fresh. - # - In user-managed mode, these configs are not validated by risedev. - # They are passed as-is to risedev-env default user for MySQL operations. - # - This is not used in RISEDEV_MYSQL_WITH_OPTIONS_COMMON. - user: root - password: "" - database: "risedev" - - # The docker image. Can be overridden to use a different version. - image: "mysql:8" - - # If set to true, data will be persisted at data/{id}. - persist-data: true - - # If `user-managed` is true, user is responsible for starting the service - # to serve at the above address and port in any way they see fit. - user-managed: false +# DEPRECATED: moved to `risedev-template.libsonnet` +template: {} From bc755c962f5d0ab151d324d14dc98f60bd5bda6a Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 17:05:31 +0800 Subject: [PATCH 04/10] more refine Signed-off-by: Bugen Zhao --- risedev-expand.libsonnet | 9 ++++--- risedev-profiles-compat.libsonnet | 4 +-- risedev-profiles.libsonnet | 17 +++++------- risedev-template.libsonnet | 38 ++++++++++++--------------- src/risedevtool/src/service_config.rs | 10 +------ 5 files changed, 31 insertions(+), 47 deletions(-) diff --git a/risedev-expand.libsonnet b/risedev-expand.libsonnet index 8a009fd765d69..ad28d7fd9ddd2 100644 --- a/risedev-expand.libsonnet +++ b/risedev-expand.libsonnet @@ -6,6 +6,7 @@ local pruneProvide = function(step) }; local mapStep = function(step, steps) + { use: std.splitLimitR(step.id, '-', 1)[0] } { [name]: if std.startsWith(name, 'provide') then [ @@ -24,9 +25,9 @@ local mapSteps = function(steps) ]; local mapRoot = function(allProfiles) - { - [name]: allProfiles[name] { steps: mapSteps(allProfiles[name].steps) } - for name in std.objectFields(allProfiles) - }; + std.mapWithKey( + function(name, profile) profile { steps: mapSteps(profile.steps) }, + allProfiles, + ); mapRoot diff --git a/risedev-profiles-compat.libsonnet b/risedev-profiles-compat.libsonnet index 8abbec644abeb..f3251fed50305 100644 --- a/risedev-profiles-compat.libsonnet +++ b/risedev-profiles-compat.libsonnet @@ -20,8 +20,6 @@ local mapStep = function(step) local mapProfile = function(name, profile) (if 'config-path' in profile then { configPath: profile['config-path'] } else {}) - { - steps: std.map(mapStep, profile.steps), - }; + { steps: std.map(mapStep, profile.steps) }; std.mapWithKey(mapProfile, profiles) diff --git a/risedev-profiles.libsonnet b/risedev-profiles.libsonnet index 9c972dd73fd85..081ab704362aa 100644 --- a/risedev-profiles.libsonnet +++ b/risedev-profiles.libsonnet @@ -1,22 +1,19 @@ local _ = import 'risedev-template.libsonnet'; { - default: { + 'default-new': { configFile: 'src/config/example.toml', steps: [ _.metaNode, _.computeNode, - // _.computeNode { port: 4588, userManaged: true }, - // _.computeNode { port: 4599 }, - // _.minio, _.frontend, ], }, - 'default-with-monitoring': $.default { - steps+: [ - _.prometheus, - _.grafana, - ], - }, + // 'default-new-with-monitoring': $['default-new'] { + // steps+: [ + // _.prometheus, + // _.grafana, + // ], + // }, } diff --git a/risedev-template.libsonnet b/risedev-template.libsonnet index 2049e88fbf6ca..246abe13413c8 100644 --- a/risedev-template.libsonnet +++ b/risedev-template.libsonnet @@ -1,6 +1,9 @@ -// The `use` field specified in the above `risedev` section will refer to the templates below. -local templates = { +// The templates used as steps in RiseDev profiles. + +{ minio: { + // Id of this instance + id: 'minio', // Advertise address of MinIO s3 endpoint address: '127.0.0.1', // Advertise port of MinIO s3 endpoint @@ -17,8 +20,6 @@ local templates = { rootPassword: 'hummockadmin', // Bucket name to store hummock information hummockBucket: 'hummock001', - // Id of this instance - id: 'minio', // Prometheus nodes used by this MinIO providePrometheus: 'prometheus*', // Max concurrent api requests. @@ -57,6 +58,8 @@ local templates = { }, computeNode: { + // Id of this instance + id: 'compute-node-' + self.port, // Compute-node advertise address address: '127.0.0.1', // Listen address @@ -65,8 +68,6 @@ local templates = { port: 5688, // Prometheus exporter listen port exporterPort: 1222, - // Id of this instance - id: 'compute-node-' + self.port, // Whether to enable async stack trace for this compute node, `off`, `on`, or `verbose`. // Considering the performance, `verbose` mode only effect under `release` profile with `debug_assertions` off. asyncStackTrace: 'verbose', @@ -92,6 +93,8 @@ local templates = { }, metaNode: { + // Id of this instance + id: 'meta-node-' + self.port, // Meta-node advertise address address: '127.0.0.1', // Meta-node listen port @@ -102,8 +105,6 @@ local templates = { dashboardPort: 5691, // Prometheus exporter listen port exporterPort: 1250, - // Id of this instance - id: 'meta-node-' + self.port, // If `user-managed` is true, this service will be started by user with the above config userManaged: false, // Etcd backend config @@ -129,14 +130,14 @@ local templates = { }, prometheus: { + // Id of this instance + id: 'prometheus', // Advertise address of Prometheus address: '127.0.0.1', // Listen port of Prometheus port: 9500, // Listen address listenAddress: self.address, - // Id of this instance - id: 'prometheus', // If `remote_write` is true, this Prometheus instance will push metrics to remote instance remoteWrite: false, // AWS region of remote write @@ -162,6 +163,8 @@ local templates = { }, frontend: { + // Id of this instance + id: 'frontend-' + self.port, // Advertise address of frontend address: '127.0.0.1', // Listen port of frontend @@ -172,8 +175,6 @@ local templates = { exporterPort: 2222, // Health check listen port healthCheckPort: 6786, - // Id of this instance - id: 'frontend-' + self.port, // Meta-nodes used by this frontend instance provideMetaNode: 'meta-node*', // Tempo used by this frontend instance @@ -183,6 +184,8 @@ local templates = { }, compactor: { + // Id of this instance + id: 'compactor-' + self.port, // Compactor advertise address address: '127.0.0.1', // Compactor listen port @@ -191,8 +194,6 @@ local templates = { listenAddress: self.address, // Prometheus exporter listen port exporterPort: 1260, - // Id of this instance - id: 'compactor-' + self.port, // Minio instances used by this compactor provideMinio: 'minio*', // Meta-nodes used by this compactor @@ -204,14 +205,14 @@ local templates = { }, grafana: { + // Id of this instance + id: 'grafana', // Listen address of Grafana listenAddress: self.address, // Advertise address of Grafana address: '127.0.0.1', // Listen port of Grafana port: 3001, - // Id of this instance - id: 'grafana', // Prometheus used by this Grafana instance providePrometheus: 'prometheus*', // Tempo used by this Grafana instance @@ -339,9 +340,4 @@ local templates = { // to serve at the above address and port in any way they see fit. userManaged: false, }, -}; - -{ - [name]: templates[name] { use: name } - for name in std.objectFields(templates) } diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 2cce28e7f4e09..32bc0b2728569 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -347,7 +347,7 @@ pub struct MySqlConfig { /// All service configuration #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "use", rename_all = "camelCase")] +#[serde(tag = "use", rename_all = "kebab-case")] // use: 'compute-node' pub enum ServiceConfig { ComputeNode(ComputeNodeConfig), MetaNode(MetaNodeConfig), @@ -369,14 +369,6 @@ pub enum ServiceConfig { Mysql(MySqlConfig), } -// impl TryFrom for ServiceConfig { -// type Error = anyhow::Error; - -// fn try_from(value: serde_json::Value) -> Result { -// todo!() -// } -// } - impl ServiceConfig { pub fn id(&self) -> &str { match self { From 919d23703005fe8f53bf7cbfbd130454e312fe8b Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 17:45:23 +0800 Subject: [PATCH 05/10] add more docs and fix config path Signed-off-by: Bugen Zhao --- risedev-expand.libsonnet | 20 +++++++++------ risedev-profiles-compat.libsonnet | 4 +++ risedev-profiles.libsonnet | 35 ++++++++++++++++++++------ risedev.jsonnet | 2 +- src/risedevtool/src/bin/risedev-dev.rs | 7 +++--- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/risedev-expand.libsonnet b/risedev-expand.libsonnet index ad28d7fd9ddd2..4867c4ad3d849 100644 --- a/risedev-expand.libsonnet +++ b/risedev-expand.libsonnet @@ -1,3 +1,5 @@ +// Post-process on a profile. + local pruneProvide = function(step) { [name]: step[name] @@ -6,16 +8,21 @@ local pruneProvide = function(step) }; local mapStep = function(step, steps) + step + // Add a kebab-case `use` field to the step from the `id` field, as the tag when deserializing. { use: std.splitLimitR(step.id, '-', 1)[0] } + // Expand all `provide-xx` fields. { - [name]: if std.startsWith(name, 'provide') then + [name]: [ + // Expand the provided steps. + // The nested `provide-xx` fields will not be used. Prune them. pruneProvide(otherStep) for otherStep in steps if std.startsWith(otherStep.id, std.rstripChars(step[name], '*')) ] - else step[name] for name in std.objectFields(step) + if std.startsWith(name, 'provide') }; local mapSteps = function(steps) @@ -24,10 +31,7 @@ local mapSteps = function(steps) for step in steps ]; -local mapRoot = function(allProfiles) - std.mapWithKey( - function(name, profile) profile { steps: mapSteps(profile.steps) }, - allProfiles, - ); +local mapProfile = function(profile) + profile { steps: mapSteps(profile.steps) }; -mapRoot +mapProfile diff --git a/risedev-profiles-compat.libsonnet b/risedev-profiles-compat.libsonnet index f3251fed50305..24c7623de2313 100644 --- a/risedev-profiles-compat.libsonnet +++ b/risedev-profiles-compat.libsonnet @@ -1,3 +1,5 @@ +// Import and transform profiles from traditional `risedev.yml` in `risedev.jsonnet`. + local yaml = importstr 'risedev.yml'; local profiles = std.parseYaml(yaml).profile; @@ -10,7 +12,9 @@ local kebabToCamel = function(s) )); local mapStep = function(step) + // Expand the full configuration for the corresponding service indicated by `use` field. _[kebabToCamel(step.use)] + // Override configuration entries with other fields. { [kebabToCamel(name)]: step[name] for name in std.objectFields(step) diff --git a/risedev-profiles.libsonnet b/risedev-profiles.libsonnet index 081ab704362aa..7e4c409da5006 100644 --- a/risedev-profiles.libsonnet +++ b/risedev-profiles.libsonnet @@ -1,8 +1,15 @@ +// Profiles for `risedev dev`. +// +// This is the next-generation version of `risedev.yml`. Through `jsonnet`, it's easier for +// developers to reuse and compose profiles. +// +// `risedev.yml` and `risedev-profiles.user.yml` are still supported now, but they will be +// eventually migrated to this file and then deprecated. + local _ = import 'risedev-template.libsonnet'; { - 'default-new': { - configFile: 'src/config/example.toml', + 'jsonnet-example': { steps: [ _.metaNode, _.computeNode, @@ -10,10 +17,22 @@ local _ = import 'risedev-template.libsonnet'; ], }, - // 'default-new-with-monitoring': $['default-new'] { - // steps+: [ - // _.prometheus, - // _.grafana, - // ], - // }, + 'jsonnet-example-with-monitoring': $['jsonnet-example'] { + // Add more steps to the base `jsonnet-example` profile. + steps+: [ + _.prometheus, + _.grafana, + ], + }, + + 'jsonnet-example-dev-frontend': { + // Specify the path to the configuration file. + configPath: 'src/config/example.toml', + steps: [ + _.metaNode, + _.computeNode, + // Override configurations for the `frontend` step. + _.frontend { userManaged: true }, + ], + }, } diff --git a/risedev.jsonnet b/risedev.jsonnet index d4d50308acb71..d990c8792d495 100644 --- a/risedev.jsonnet +++ b/risedev.jsonnet @@ -7,6 +7,6 @@ local allProfiles = profiles + compatProfiles; local profile = std.extVar('profile'); if profile in allProfiles then - expand(allProfiles)[profile] { profile: profile } + expand(allProfiles[profile]) else error 'unknown profile ' + profile diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index c2e5247d94662..eed6583e08a9d 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -340,9 +340,9 @@ fn task_main( } #[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] struct Input { - profile: String, - config_file: Option, + config_path: Option, steps: Vec, } @@ -378,8 +378,7 @@ fn main() -> Result<()> { )?; let Input { - profile: task_name, - config_file: config_path, + config_path, steps: services, }: Input = serde_json::from_slice(&input).context("failed to parse input")?; From b97d5559b209ed03e381cbaf2a8654052f055f71 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 17:56:05 +0800 Subject: [PATCH 06/10] move files Signed-off-by: Bugen Zhao --- risedev.jsonnet | 12 ------------ .../risedevtool/jsonnet/expand.libsonnet | 0 src/risedevtool/jsonnet/main.jsonnet | 14 ++++++++++++++ .../risedevtool/jsonnet/profiles-compat.libsonnet | 4 ++-- src/risedevtool/src/bin/risedev-dev.rs | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) delete mode 100644 risedev.jsonnet rename risedev-expand.libsonnet => src/risedevtool/jsonnet/expand.libsonnet (100%) create mode 100644 src/risedevtool/jsonnet/main.jsonnet rename risedev-profiles-compat.libsonnet => src/risedevtool/jsonnet/profiles-compat.libsonnet (88%) diff --git a/risedev.jsonnet b/risedev.jsonnet deleted file mode 100644 index d990c8792d495..0000000000000 --- a/risedev.jsonnet +++ /dev/null @@ -1,12 +0,0 @@ -local expand = import 'risedev-expand.libsonnet'; - -local compatProfiles = import 'risedev-profiles-compat.libsonnet'; -local profiles = import 'risedev-profiles.libsonnet'; -local allProfiles = profiles + compatProfiles; - -local profile = std.extVar('profile'); - -if profile in allProfiles then - expand(allProfiles[profile]) -else - error 'unknown profile ' + profile diff --git a/risedev-expand.libsonnet b/src/risedevtool/jsonnet/expand.libsonnet similarity index 100% rename from risedev-expand.libsonnet rename to src/risedevtool/jsonnet/expand.libsonnet diff --git a/src/risedevtool/jsonnet/main.jsonnet b/src/risedevtool/jsonnet/main.jsonnet new file mode 100644 index 0000000000000..956c5ccdb7676 --- /dev/null +++ b/src/risedevtool/jsonnet/main.jsonnet @@ -0,0 +1,14 @@ +// Entrypoint for selecting a profile and expanding it. + +local expand = import 'expand.libsonnet'; + +local profiles = import '../../../risedev-profiles.libsonnet'; +local compatProfiles = import 'profiles-compat.libsonnet'; +local allProfiles = profiles + compatProfiles; + +local profile = std.extVar('profile'); + +if profile in allProfiles then + expand(allProfiles[profile]) +else + error 'unknown profile ' + profile diff --git a/risedev-profiles-compat.libsonnet b/src/risedevtool/jsonnet/profiles-compat.libsonnet similarity index 88% rename from risedev-profiles-compat.libsonnet rename to src/risedevtool/jsonnet/profiles-compat.libsonnet index 24c7623de2313..035d62324b286 100644 --- a/risedev-profiles-compat.libsonnet +++ b/src/risedevtool/jsonnet/profiles-compat.libsonnet @@ -1,9 +1,9 @@ // Import and transform profiles from traditional `risedev.yml` in `risedev.jsonnet`. -local yaml = importstr 'risedev.yml'; +local yaml = importstr '../../../risedev.yml'; local profiles = std.parseYaml(yaml).profile; -local _ = import 'risedev-template.libsonnet'; +local _ = import '../../../risedev-template.libsonnet'; local kebabToCamel = function(s) std.join('', std.mapWithIndex( diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index eed6583e08a9d..a919778e376c9 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -358,7 +358,7 @@ fn main() -> Result<()> { .unwrap_or_else(|| "default".to_string()); let json = std::process::Command::new("rsjsonnet") - .arg("risedev.jsonnet") + .arg("src/risedevtool/jsonnet/main.jsonnet") .arg("-V") .arg(format!("profile={task_name}")) .output() From 43b6ff573fdf645c6ae2a9a0604e00ad7bd58d05 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 18:00:20 +0800 Subject: [PATCH 07/10] remove config expanders Signed-off-by: Bugen Zhao --- Cargo.lock | 11 - src/risedevtool/Cargo.toml | 2 - src/risedevtool/src/bin/risedev-compose.rs | 305 +----------------- src/risedevtool/src/bin/risedev-dev.rs | 10 +- src/risedevtool/src/config.rs | 193 ----------- src/risedevtool/src/config/dollar_expander.rs | 125 ------- src/risedevtool/src/config/id_expander.rs | 140 -------- .../src/config/provide_expander.rs | 145 --------- src/risedevtool/src/config/use_expander.rs | 149 --------- src/risedevtool/src/lib.rs | 2 - src/risedevtool/src/service_config.rs | 26 -- 11 files changed, 6 insertions(+), 1102 deletions(-) delete mode 100644 src/risedevtool/src/config.rs delete mode 100644 src/risedevtool/src/config/dollar_expander.rs delete mode 100644 src/risedevtool/src/config/id_expander.rs delete mode 100644 src/risedevtool/src/config/provide_expander.rs delete mode 100644 src/risedevtool/src/config/use_expander.rs diff --git a/Cargo.lock b/Cargo.lock index 28aa7b737e349..6fb107afd604e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9942,13 +9942,11 @@ dependencies = [ "serde", "serde_json", "serde_with", - "serde_yaml", "tempfile", "thiserror-ext", "tracing", "tracing-subscriber", "workspace-hack", - "yaml-rust", ] [[package]] @@ -16358,15 +16356,6 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "0.5.1" diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml index 78ec4b5a63e45..fb91712be0754 100644 --- a/src/risedevtool/Cargo.toml +++ b/src/risedevtool/Cargo.toml @@ -33,7 +33,6 @@ reqwest = { version = "0.12.2", features = ["blocking"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_with = "3" -serde_yaml = "0.9" tempfile = "3" thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ @@ -48,7 +47,6 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ tracing = "0.1" tracing-subscriber = "0.3" workspace-hack = { path = "../workspace-hack" } -yaml-rust = "0.4" [lints] workspace = true diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index 46a0314846c92..388158a0e21f9 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -12,309 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; -use std::io::Read; -use std::path::Path; - -use anyhow::{anyhow, Result}; -use clap::Parser; -use console::style; -use fs_err::{self, File}; -use itertools::Itertools; -use risedev::{ - compose_deploy, generate_risedev_env, Compose, ComposeConfig, ComposeDeployConfig, ComposeFile, - ComposeService, ComposeVolume, ConfigExpander, DockerImageConfig, ServiceConfig, - RISEDEV_CONFIG_FILE, -}; -use serde::Deserialize; - -#[derive(Parser)] -#[clap(author, version, about, long_about = None)] -#[clap(propagate_version = true)] -pub struct RiseDevComposeOpts { - #[clap(short, long)] - directory: String, - - #[clap(default_value = "compose")] - profile: String, - - /// Whether to generate deployment script. If enabled, network mode will be set to host, a - /// deploy.sh will be generated. - #[clap(long)] - deploy: bool, -} - -fn load_docker_image_config( - risedev_config: &str, - override_risingwave_image: Option<&String>, -) -> Result { - #[derive(Deserialize)] - struct ConfigInRiseDev { - compose: DockerImageConfig, - } - let mut config: ConfigInRiseDev = serde_yaml::from_str(risedev_config)?; - if let Some(override_risingwave_image) = override_risingwave_image { - config.compose.risingwave = override_risingwave_image.to_string(); - } - Ok(config.compose) -} +use anyhow::{bail, Result}; fn main() -> Result<()> { - let opts = RiseDevComposeOpts::parse(); - - let (risedev_config, compose_deploy_config, rw_config_path) = if opts.deploy { - let compose_deploy_config = { - let mut content = String::new(); - File::open("risedev-compose.yml")?.read_to_string(&mut content)?; - content - }; - let compose_deploy_config: ComposeDeployConfig = - serde_yaml::from_str(&compose_deploy_config)?; - let extra_info = compose_deploy_config - .instances - .iter() - .map(|i| (format!("dns-host:{}", i.id), i.dns_host.clone())) - .chain( - compose_deploy_config - .risedev_extra_args - .iter() - .map(|(k, v)| (k.clone(), v.clone())), - ) - .collect(); - - let (config_path, expanded_config) = - ConfigExpander::expand_with_extra_info(".", &opts.profile, extra_info)?; - (expanded_config, Some(compose_deploy_config), config_path) - } else { - let (config_path, expanded_config) = ConfigExpander::expand(".", &opts.profile)?; - (expanded_config, None, config_path) - }; - - let compose_config = ComposeConfig { - image: load_docker_image_config( - &fs_err::read_to_string(RISEDEV_CONFIG_FILE)?, - compose_deploy_config - .as_ref() - .and_then(|x| x.risingwave_image_override.as_ref()), - )?, - config_directory: opts.directory.clone(), - rw_config_path, - }; - - let services = ConfigExpander::deserialize(&risedev_config)?; - - let mut compose_services: BTreeMap> = BTreeMap::new(); - let mut service_on_node: BTreeMap = BTreeMap::new(); - let mut volumes = BTreeMap::new(); - - let mut log_buffer = String::new(); - use std::fmt::Write; - - for service in &services { - let step = service.id(); - - let compose_deploy_config = compose_deploy_config.as_ref(); - let (address, mut compose) = match service { - ServiceConfig::Minio(c) => { - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Etcd(c) => { - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Sqlite(_) => continue, - ServiceConfig::Prometheus(c) => { - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::ComputeNode(c) => { - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::MetaNode(c) => { - if opts.deploy { - let public_ip = &compose_deploy_config - .unwrap() - .lookup_instance_by_host(&c.address) - .public_ip; - writeln!( - log_buffer, - "-- Dashboard --\nuse VSCode to forward {} from {}\nor use {}\n", - style(format!("{}", c.dashboard_port)).green(), - style(format!("ubuntu@{}", public_ip)).green(), - style(format!( - "ssh -N -L {}:localhost:{} ubuntu@{}", - c.dashboard_port, c.dashboard_port, public_ip - )) - .green() - )?; - } - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Frontend(c) => { - if opts.deploy { - let arg = format!("--frontend {} --frontend-port {}", c.address, c.port); - writeln!( - log_buffer, - "-- Frontend --\nAccess inside cluster: {}\ntpch-bench args: {}\n", - style(format!( - "psql -d dev -h {} -p {} -U root", - c.address, c.port - )) - .green(), - style(&arg).green() - )?; - fs_err::write( - Path::new(&opts.directory).join("tpch-bench-args-frontend"), - arg, - )?; - } - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Compactor(c) => (c.address.clone(), c.compose(&compose_config)?), - ServiceConfig::Grafana(c) => { - if opts.deploy { - let public_ip = &compose_deploy_config - .unwrap() - .lookup_instance_by_host(&c.address) - .public_ip; - writeln!( - log_buffer, - "-- Grafana --\nuse VSCode to forward {} from {}\nor use {}\n", - style(format!("{}", c.port)).green(), - style(format!("ubuntu@{}", public_ip)).green(), - style(format!( - "ssh -N -L {}:localhost:{} ubuntu@{}", - c.port, c.port, public_ip - )) - .green() - )?; - } - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Tempo(c) => (c.address.clone(), c.compose(&compose_config)?), - ServiceConfig::Kafka(_) => { - return Err(anyhow!("not supported, please use redpanda instead")) - } - ServiceConfig::Pubsub(_) => { - return Err(anyhow!("not supported, please use redpanda instead")) - } - ServiceConfig::ZooKeeper(_) => { - return Err(anyhow!("not supported, please use redpanda instead")) - } - ServiceConfig::Opendal(_) => continue, - ServiceConfig::AwsS3(_) => continue, - ServiceConfig::RedPanda(c) => { - if opts.deploy { - let arg = format!("--kafka-addr {}:{}", c.address, c.internal_port); - writeln!( - log_buffer, - "-- Redpanda --\ntpch-bench: {}\n", - style(&arg).green() - )?; - fs_err::write( - Path::new(&opts.directory).join("tpch-bench-args-kafka"), - arg, - )?; - } - volumes.insert(c.id.clone(), ComposeVolume::default()); - (c.address.clone(), c.compose(&compose_config)?) - } - ServiceConfig::Redis(_) | ServiceConfig::Mysql(_) => { - return Err(anyhow!("not supported")) - } - }; - compose.container_name = service.id().to_string(); - if opts.deploy { - compose.network_mode = Some("host".into()); - compose.depends_on = vec![]; - } - compose_services - .entry(address.clone()) - .or_default() - .insert(step.to_string(), compose); - service_on_node.insert(step.to_string(), address); - } - - if opts.deploy { - for (node, services) in &compose_services { - let mut node_volumes = BTreeMap::new(); - services.keys().for_each(|k| { - if let Some(v) = volumes.get(k) { - node_volumes.insert(k.clone(), v.clone()); - } - }); - let compose_file = ComposeFile { - version: "3".into(), - services: services.clone(), - volumes: node_volumes, - name: format!("risingwave-{}", opts.profile), - }; - - let yaml = serde_yaml::to_string(&compose_file)?; - - let ec2_instance = compose_deploy_config - .as_ref() - .unwrap() - .lookup_instance_by_host(node); - if ec2_instance.r#type == "meta" { - let public_ip = &ec2_instance.public_ip; - writeln!( - log_buffer, - "-- Meta Node --\nLogin to meta node by {}\nor using VSCode {}\n", - style(format!("ssh ubuntu@{}", public_ip)).green(), - style(format!( - "code --remote ssh-remote+ubuntu@{} ", - public_ip - )) - .green() - )?; - } - - fs_err::write( - Path::new(&opts.directory).join(format!("{}.yml", node)), - yaml, - )?; - } - - if let Some(env) = Some(generate_risedev_env(&services)).filter(|x| !x.is_empty()) { - writeln!(log_buffer, "-- risedev-env --\n{}\n", style(env).green())?; - } - - compose_deploy( - Path::new(&opts.directory), - &services.iter().map(|s| s.id().to_string()).collect_vec(), - &compose_deploy_config.as_ref().unwrap().instances, - &compose_config, - &service_on_node, - )?; - - println!("\n{}", log_buffer); - - fs_err::write( - Path::new(&opts.directory).join("_message.partial.sh"), - log_buffer, - )?; - } else { - let mut services = BTreeMap::new(); - for (_, s) in compose_services { - for (k, v) in s { - services.insert(k, v); - } - } - let compose_file = ComposeFile { - version: "3".into(), - services, - volumes, - name: format!("risingwave-{}", opts.profile), - }; - - let yaml = serde_yaml::to_string(&compose_file)?; - - fs_err::write(Path::new(&opts.directory).join("docker-compose.yml"), yaml)?; - } - - Ok(()) + bail!("risedev-compose is deprecated") } diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index a919778e376c9..c132a4fedb3eb 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -26,16 +26,14 @@ use fs_err::OpenOptions; use indicatif::ProgressBar; use risedev::util::{complete_spin, fail_spin}; use risedev::{ - generate_risedev_env, preflight_check, CompactorService, ComputeNodeService, ConfigExpander, - ConfigureTmuxTask, DummyService, EnsureStopService, ExecuteContext, FrontendService, - GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PrometheusService, - PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, TempoService, ZooKeeperService, - RISEDEV_NAME, + generate_risedev_env, preflight_check, CompactorService, ComputeNodeService, ConfigureTmuxTask, + DummyService, EnsureStopService, ExecuteContext, FrontendService, GrafanaService, KafkaService, + MetaNodeService, MinioService, MySqlService, PrometheusService, PubsubService, RedisService, + ServiceConfig, SqliteConfig, Task, TempoService, ZooKeeperService, RISEDEV_NAME, }; use serde::{Deserialize, Serialize}; use tempfile::tempdir; use thiserror_ext::AsReport; -use yaml_rust::YamlEmitter; #[derive(Default)] pub struct ProgressManager { diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs deleted file mode 100644 index cfbae23fc9371..0000000000000 --- a/src/risedevtool/src/config.rs +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::path::Path; - -use anyhow::{anyhow, bail, Result}; -use itertools::Itertools; -use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; - -use crate::ServiceConfig; - -mod dollar_expander; -mod id_expander; -mod provide_expander; -mod use_expander; -use dollar_expander::DollarExpander; -use id_expander::IdExpander; -use provide_expander::ProvideExpander; -use use_expander::UseExpander; - -/// The main configuration file name. -pub const RISEDEV_CONFIG_FILE: &str = "risedev.yml"; -/// The extra user profiles file name. -pub const RISEDEV_USER_PROFILES_FILE: &str = "risedev-profiles.user.yml"; - -pub struct ConfigExpander; - -impl ConfigExpander { - /// Load a single document YAML file. - fn load_yaml(path: impl AsRef) -> Result { - let path = path.as_ref(); - let content = fs_err::read_to_string(path)?; - let [config]: [_; 1] = YamlLoader::load_from_str(&content)? - .try_into() - .map_err(|_| anyhow!("expect `{}` to have only one section", path.display()))?; - Ok(config) - } - - /// Transforms `risedev.yml` and `risedev-profiles.user.yml` to a fully expanded yaml file. - /// - /// # Arguments - /// - /// * `root` is the root directory of these YAML files. - /// * `profile` is the selected config profile called by `risedev dev `. It is one of - /// the keys in the `profile` section. - /// - /// # Returns - /// - /// A pair of `config_path` and expanded steps (items in `{profile}.steps` section in YAML) - pub fn expand(root: impl AsRef, profile: &str) -> Result<(Option, Yaml)> { - Self::expand_with_extra_info(root, profile, HashMap::new()) - } - - /// See [`ConfigExpander::expand`] for other information. - /// - /// # Arguments - /// - /// - `extra_info` is additional variables for variable expansion by [`DollarExpander`]. - pub fn expand_with_extra_info( - root: impl AsRef, - profile: &str, - extra_info: HashMap, - ) -> Result<(Option, Yaml)> { - let global_path = root.as_ref().join(RISEDEV_CONFIG_FILE); - let global_yaml = Self::load_yaml(global_path)?; - let global_config = global_yaml - .as_hash() - .ok_or_else(|| anyhow!("expect config to be a hashmap"))?; - - let all_profile_section = { - let mut all = global_config - .get(&Yaml::String("profile".to_string())) - .ok_or_else(|| anyhow!("expect `profile` section"))? - .as_hash() - .ok_or_else(|| anyhow!("expect `profile` section to be a hashmap"))? - .to_owned(); - - // Add user profiles if exists. - let user_profiles_path = root.as_ref().join(RISEDEV_USER_PROFILES_FILE); - if user_profiles_path.is_file() { - let yaml = Self::load_yaml(user_profiles_path)?; - let map = yaml.as_hash().ok_or_else(|| { - anyhow!("expect `{RISEDEV_USER_PROFILES_FILE}` to be a hashmap") - })?; - for (k, v) in map { - if all.insert(k.clone(), v.clone()).is_some() { - bail!( - "find duplicated config key `{k:?}` in `{RISEDEV_USER_PROFILES_FILE}`" - ); - } - } - } - - all - }; - - let template_section = global_config - .get(&Yaml::String("template".to_string())) - .ok_or_else(|| anyhow!("expect `profile` section"))?; - - let profile_section = all_profile_section - .get(&Yaml::String(profile.to_string())) - .ok_or_else(|| anyhow!("profile '{}' not found", profile))? - .as_hash() - .ok_or_else(|| anyhow!("expect `profile` section to be a hashmap"))?; - - let config_path = profile_section - .get(&Yaml::String("config-path".to_string())) - .and_then(|s| s.as_str()) - .map(|s| s.to_string()); - - let steps = profile_section - .get(&Yaml::String("steps".to_string())) - .ok_or_else(|| anyhow!("expect `steps` section"))? - .clone(); - - let steps = UseExpander::new(template_section)?.visit(steps)?; - let steps = DollarExpander::new(extra_info).visit(steps)?; - let steps = IdExpander::new(&steps)?.visit(steps)?; - let steps = ProvideExpander::new(&steps)?.visit(steps)?; - - Ok((config_path, steps)) - } - - /// Parses the expanded yaml into [`ServiceConfig`]s. - /// The order is the same as the original array's order. - pub fn deserialize(expanded_config: &Yaml) -> Result> { - let steps = expanded_config - .as_vec() - .ok_or_else(|| anyhow!("expect steps to be an array"))?; - let config: Vec = steps - .iter() - .map(|step| { - let use_type = step - .as_hash() - .ok_or_else(|| anyhow!("expect step to be a hashmap"))?; - let use_type = use_type - .get(&Yaml::String("use".to_string())) - .ok_or_else(|| anyhow!("expect `use` in step"))?; - let use_type = use_type - .as_str() - .ok_or_else(|| anyhow!("expect `use` to be a string"))? - .to_string(); - let mut out_str = String::new(); - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(step)?; - let result = match use_type.as_str() { - "minio" => ServiceConfig::Minio(serde_yaml::from_str(&out_str)?), - "etcd" => ServiceConfig::Etcd(serde_yaml::from_str(&out_str)?), - "sqlite" => ServiceConfig::Sqlite(serde_yaml::from_str(&out_str)?), - "frontend" => ServiceConfig::Frontend(serde_yaml::from_str(&out_str)?), - "compactor" => ServiceConfig::Compactor(serde_yaml::from_str(&out_str)?), - "compute-node" => ServiceConfig::ComputeNode(serde_yaml::from_str(&out_str)?), - "meta-node" => ServiceConfig::MetaNode(serde_yaml::from_str(&out_str)?), - "prometheus" => ServiceConfig::Prometheus(serde_yaml::from_str(&out_str)?), - "grafana" => ServiceConfig::Grafana(serde_yaml::from_str(&out_str)?), - "tempo" => ServiceConfig::Tempo(serde_yaml::from_str(&out_str)?), - "opendal" => ServiceConfig::Opendal(serde_yaml::from_str(&out_str)?), - "aws-s3" => ServiceConfig::AwsS3(serde_yaml::from_str(&out_str)?), - "kafka" => ServiceConfig::Kafka(serde_yaml::from_str(&out_str)?), - "pubsub" => ServiceConfig::Pubsub(serde_yaml::from_str(&out_str)?), - "redis" => ServiceConfig::Redis(serde_yaml::from_str(&out_str)?), - "zookeeper" => ServiceConfig::ZooKeeper(serde_yaml::from_str(&out_str)?), - "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?), - "mysql" => ServiceConfig::Mysql(serde_yaml::from_str(&out_str)?), - other => return Err(anyhow!("unsupported use type: {}", other)), - }; - Ok(result) - }) - .try_collect()?; - - let mut services = HashMap::new(); - for x in &config { - let id = x.id().to_string(); - if services.insert(id.clone(), x).is_some() { - return Err(anyhow!("duplicate id: {}", id)); - } - } - Ok(config) - } -} diff --git a/src/risedevtool/src/config/dollar_expander.rs b/src/risedevtool/src/config/dollar_expander.rs deleted file mode 100644 index 47f302140309a..0000000000000 --- a/src/risedevtool/src/config/dollar_expander.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; - -use anyhow::{anyhow, Result}; -use regex::Regex; -use yaml_rust::Yaml; - -/// Expands `x-${port}` to `x-2333`. -pub struct DollarExpander { - re: Regex, - extra_info: HashMap, -} - -fn yaml_to_string(y: &Yaml) -> Result { - match y { - Yaml::Integer(x) => Ok(format!("{}", x)), - Yaml::String(x) => Ok(x.clone()), - _ => Err(anyhow!("{:?} cannot be converted to string", y)), - } -} - -impl DollarExpander { - pub fn new(extra_info: HashMap) -> Self { - Self { - re: Regex::new(r"\$\{(.*?)\}").unwrap(), - extra_info, - } - } - - pub fn visit(&mut self, y: Yaml) -> Result { - match y { - Yaml::Hash(y) => { - let mut ny = y.clone(); - for (_, v) in &mut ny { - let result = if let Some(v) = v.as_str() { - let mut target = String::new(); - let mut last_location = 0; - for cap in self.re.captures_iter(v) { - let cap = cap.get(1).unwrap(); - let name = cap.as_str(); - let value = if let Some(item) = y.get(&Yaml::String(name.to_string())) { - yaml_to_string(item)? - } else if let Some(item) = self.extra_info.get(name) { - item.clone() - } else { - return Err(anyhow!("{} not found in {:?}", name, y)); - }; - target += &v[last_location..(cap.start() - 2)]; // ignore `${` - target += &value; - last_location = cap.end() + 1; // ignore `}` - } - target += &v[last_location..v.len()]; - Yaml::String(target) - } else { - self.visit(v.clone())? - }; - *v = result; - } - Ok(Yaml::Hash(ny)) - } - Yaml::Array(mut yv) => { - for y in &mut yv { - *y = self.visit(y.clone())?; - } - Ok(Yaml::Array(yv)) - } - other => Ok(other), - } - } -} - -#[cfg(test)] -mod tests { - use yaml_rust::YamlLoader; - - use super::*; - - #[test] - fn test_expand_dollar() { - let yaml = YamlLoader::load_from_str( - " -a: - b: - c: - d: \"${x}${y},${test:key}\" - x: 2333 - y: 2334 - ", - ) - .unwrap() - .remove(0); - let yaml_result = YamlLoader::load_from_str( - " - a: - b: - c: - d: \"23332334,value\" - x: 2333 - y: 2334 - ", - ) - .unwrap() - .remove(0); - let mut visitor = DollarExpander::new( - vec![("test:key".to_string(), "value".to_string())] - .into_iter() - .collect(), - ); - - assert_eq!(visitor.visit(yaml).unwrap(), yaml_result); - } -} diff --git a/src/risedevtool/src/config/id_expander.rs b/src/risedevtool/src/config/id_expander.rs deleted file mode 100644 index 01eae93843f5c..0000000000000 --- a/src/risedevtool/src/config/id_expander.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::{anyhow, Result}; -use regex::Regex; -use yaml_rust::Yaml; - -/// Expands `x-*` to `x-2333` by finding id in map. -pub struct IdExpander { - ids: Vec, -} - -impl IdExpander { - pub fn new(yaml: &Yaml) -> Result { - let yaml = yaml - .as_vec() - .ok_or_else(|| anyhow!("Not an array: {:?}", yaml))?; - let mut ids = vec![]; - for item in yaml { - if let Some(item) = item.as_hash() { - let id = item - .get(&Yaml::String("id".to_string())) - .ok_or_else(|| anyhow!("Missing id field: {:?}", item))? - .as_str() - .ok_or_else(|| anyhow!("Id isn't a string: {:?}", item))?; - ids.push(id.to_string()); - } else { - return Err(anyhow!("Not an hashmap: {:?}", item)); - } - } - Ok(Self { ids }) - } - - pub fn visit_vec(&mut self, mut yv: Vec) -> Result> { - for y in &mut yv { - *y = self.visit(y.clone())?; - } - Ok(yv) - } - - pub fn visit(&mut self, y: Yaml) -> Result { - match y { - Yaml::Hash(y) => { - let mut ny = y; - for (_, v) in &mut ny { - let result = if let Some(v) = v.as_str() { - if let Some((before, after)) = v.split_once('*') { - let regex = Regex::new(&format!("^{}(.*){}$", before, after))?; - let mut matched_ids = vec![]; - for id in &self.ids { - if regex.is_match(id) { - matched_ids.push(Yaml::String(id.clone())); - } - } - Yaml::Array(matched_ids) - } else { - Yaml::String(v.to_string()) - } - } else { - self.visit(v.clone())? - }; - *v = result; - } - Ok(Yaml::Hash(ny)) - } - Yaml::Array(y) => Ok(Yaml::Array(self.visit_vec(y)?)), - other => Ok(other), - } - } -} - -#[cfg(test)] -mod tests { - use yaml_rust::YamlLoader; - - use super::*; - - #[test] - fn test_expand_id() { - let yaml = YamlLoader::load_from_str( - " -- id: b-233 - provide: \"a-*\" -- id: a-233 - ", - ) - .unwrap(); - let yaml_result = YamlLoader::load_from_str( - " -- id: b-233 - provide: [\"a-233\"] -- id: a-233 - ", - ) - .unwrap(); - let mut visitor = IdExpander::new(&yaml[0]).unwrap(); - - assert_eq!(visitor.visit_vec(yaml).unwrap(), yaml_result); - } - - #[test] - fn test_expand_id_array() { - let yaml = YamlLoader::load_from_str( - " -- id: b-233 - provide: \"a-*\" -- id: a-233 -- id: a-234 -- id: aa-233 - ", - ) - .unwrap() - .remove(0); - let yaml_result = YamlLoader::load_from_str( - " -- id: b-233 - provide: [\"a-233\", \"a-234\"] -- id: a-233 -- id: a-234 -- id: aa-233 - ", - ) - .unwrap() - .remove(0); - let mut visitor = IdExpander::new(&yaml).unwrap(); - - assert_eq!(visitor.visit(yaml).unwrap(), yaml_result); - } -} diff --git a/src/risedevtool/src/config/provide_expander.rs b/src/risedevtool/src/config/provide_expander.rs deleted file mode 100644 index 62603b2b625f0..0000000000000 --- a/src/risedevtool/src/config/provide_expander.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; - -use anyhow::{anyhow, Result}; -use itertools::Itertools; -use yaml_rust::Yaml; - -/// Expands `provide-xxx: ["a", "b", "c"]` to their corresponding IDs. -pub struct ProvideExpander { - all_items: HashMap, -} - -impl ProvideExpander { - pub fn new(y: &Yaml) -> Result { - let y = y.as_vec().ok_or_else(|| anyhow!("expect an array"))?; - let mut all_items = HashMap::new(); - for v in y { - let v = v.as_hash().ok_or_else(|| anyhow!("expect a hashmap"))?; - let id = v - .get(&Yaml::String("id".into())) - .ok_or_else(|| anyhow!("missing id field"))?; - let id = id - .as_str() - .ok_or_else(|| anyhow!("expect id to be a string"))?; - all_items.insert(id.to_string(), Yaml::Hash(v.clone())); - } - Ok(Self { - all_items: Self::remove_provide(all_items)?, - }) - } - - fn remove_provide(all_items: HashMap) -> Result> { - let all_items = all_items.into_iter().map(|(k, v)| { - let v = v.into_hash().ok_or_else(|| anyhow!("expect a hashmap"))?; - let v = v - .into_iter() - .filter(|(k, _)| { - if let Some(k) = k.as_str() { - !k.starts_with("provide-") - } else { - true - } - }) - .collect(); - Ok::<_, anyhow::Error>((k, Yaml::Hash(v))) - }); - all_items.try_collect() - } - - pub fn visit(&mut self, yaml: Yaml) -> Result { - let yaml = yaml - .into_vec() - .ok_or_else(|| anyhow!("expect an array"))? - .into_iter() - .map(|yaml| { - let map = yaml - .into_hash() - .ok_or_else(|| anyhow!("expect a hashmap"))?; - let map = map.into_iter().map(|(k, v)| { - if let Some(k) = k.as_str() - && k.starts_with("provide-") - { - let array = v - .as_vec() - .ok_or_else(|| anyhow!("expect an array of provide-"))?; - let array = array.iter().map(|item| { - let item = item - .as_str() - .ok_or_else(|| anyhow!("expect a string from provide"))?; - Ok::<_, anyhow::Error>( - self.all_items - .get(item) - .ok_or_else(|| anyhow!("{} not found", item))? - .clone(), - ) - }); - return Ok::<_, anyhow::Error>(( - Yaml::String(k.to_string()), - Yaml::Array(array.try_collect()?), - )); - } - Ok::<_, anyhow::Error>((k, v)) - }); - Ok::<_, anyhow::Error>(Yaml::Hash(map.try_collect()?)) - }); - Ok(Yaml::Array(yaml.try_collect()?)) - } -} - -#[cfg(test)] -mod tests { - use yaml_rust::YamlLoader; - - use super::*; - - #[test] - fn test_expand_provide() { - let source = YamlLoader::load_from_str( - " -- provide-b: [\"b\"] - test_field: a - id: a -- provide-a: [\"a\"] - test_field: a - id: b - ", - ) - .unwrap() - .remove(0); - - let expected_result = YamlLoader::load_from_str( - " -- provide-b: - - test_field: a - id: b - test_field: a - id: a -- provide-a: - - test_field: a - id: a - test_field: a - id: b - ", - ) - .unwrap() - .remove(0); - - let mut visitor = ProvideExpander::new(&source).unwrap(); - - assert_eq!(visitor.visit(source).unwrap(), expected_result); - } -} diff --git a/src/risedevtool/src/config/use_expander.rs b/src/risedevtool/src/config/use_expander.rs deleted file mode 100644 index 49386e5e983f9..0000000000000 --- a/src/risedevtool/src/config/use_expander.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; - -use anyhow::{anyhow, Result}; -use itertools::Itertools; -use yaml_rust::{yaml, Yaml}; - -/// Expands `use: xxx` from the template. -pub struct UseExpander { - template: HashMap, -} - -impl UseExpander { - pub fn new(template: &Yaml) -> Result { - let ytm = template - .as_hash() - .ok_or_else(|| anyhow!("template is not a hashmap"))?; - let mut template = HashMap::new(); - for (k, v) in ytm { - let k = k - .as_str() - .ok_or_else(|| anyhow!("key {:?} is not a string", k))?; - let v = v - .as_hash() - .ok_or_else(|| anyhow!("expect value to be a hashmap"))?; - template.insert(k.to_string(), v.clone()); - } - Ok(Self { template }) - } - - /// Overrides values in `default` with values from `provided`. - fn merge(use_id: &str, default: &yaml::Hash, provided: &yaml::Hash) -> yaml::Hash { - let mut result = yaml::Hash::new(); - // put `use` as the first element to make the generated yaml more readable. - result.insert(Yaml::String("use".into()), Yaml::String(use_id.into())); - result.extend(default.clone()); - for (k, new_v) in provided { - match result.get_mut(k) { - Some(v) => { - // update the value, but do not change the order. - *v = new_v.clone() - } - None => { - // For keys not defined in the template (optional keys), we just append them - // here. It may be rejected later when deserializing to - // specific `ServiceConfig` if it's invalid. - result.insert(k.clone(), new_v.clone()); - } - }; - } - result - } - - pub fn visit(&mut self, yaml: Yaml) -> Result { - let yaml = yaml - .as_vec() - .ok_or_else(|| anyhow!("expect an array for use"))?; - let array = yaml.iter().map(|item| { - let map = item - .as_hash() - .ok_or_else(|| anyhow!("expect a hashmap for use"))?; - - let use_id_yaml = map - .get(&Yaml::String("use".into())) - .ok_or_else(|| anyhow!("expect `use` in hashmap"))?; - let use_id = use_id_yaml - .as_str() - .ok_or_else(|| anyhow!("expect `use` to be a string"))?; - let use_data = self - .template - .get(use_id) - .ok_or_else(|| anyhow!("use source {} not found", use_id))?; - - if map.get(&Yaml::String("config-path".into())).is_some() { - return Err(anyhow!( - "`config-path` should not be put inside a `use` step. \ - Put `config-path` as a property parallel to `steps` instead." - )); - } - - Ok::<_, anyhow::Error>(Yaml::Hash(Self::merge(use_id, use_data, map))) - }); - Ok(Yaml::Array(array.try_collect()?)) - } -} - -#[cfg(test)] -mod tests { - use yaml_rust::YamlLoader; - - use super::*; - #[test] - fn test_expand_use() { - let template = YamlLoader::load_from_str( - " -test: - a: 2333 - b: 23333 -test2: - a: 23333 - b: 233333 - ", - ) - .unwrap() - .remove(0); - - let use_expand = YamlLoader::load_from_str( - " -- use: test - a: 23333 - c: 23333 -- use: test2 - d: 23333", - ) - .unwrap() - .remove(0); - - let expected_result = YamlLoader::load_from_str( - " -- use: test - a: 23333 - b: 23333 - c: 23333 -- use: test2 - a: 23333 - b: 233333 - d: 23333", - ) - .unwrap() - .remove(0); - - let mut visitor = UseExpander::new(&template).unwrap(); - - assert_eq!(visitor.visit(use_expand).unwrap(), expected_result); - } -} diff --git a/src/risedevtool/src/lib.rs b/src/risedevtool/src/lib.rs index 57294e5a7eafa..ee228110bab03 100644 --- a/src/risedevtool/src/lib.rs +++ b/src/risedevtool/src/lib.rs @@ -17,8 +17,6 @@ #![feature(let_chains)] #![feature(lint_reasons)] -mod config; -pub use config::*; mod config_gen; pub use config_gen::*; mod preflight_check; diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 32bc0b2728569..7fcbb17053a9a 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -439,29 +439,3 @@ impl ServiceConfig { } } } - -mod string { - use std::fmt::Display; - use std::str::FromStr; - - use serde::{de, Deserialize, Deserializer, Serializer}; - - pub fn serialize(value: &T, serializer: S) -> Result - where - T: Display, - S: Serializer, - { - serializer.collect_str(value) - } - - pub fn deserialize<'de, T, D>(deserializer: D) -> Result - where - T: FromStr, - T::Err: Display, - D: Deserializer<'de>, - { - String::deserialize(deserializer)? - .parse() - .map_err(de::Error::custom) - } -} From 138203046773f08d4fd1ee699a9ab4717fe5647f Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 18:07:50 +0800 Subject: [PATCH 08/10] refine more Signed-off-by: Bugen Zhao --- src/risedevtool/jsonnet/expand.libsonnet | 3 +++ src/risedevtool/src/bin/risedev-dev.rs | 5 +++-- src/risedevtool/src/service_config.rs | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/risedevtool/jsonnet/expand.libsonnet b/src/risedevtool/jsonnet/expand.libsonnet index 4867c4ad3d849..abd6945539e7a 100644 --- a/src/risedevtool/jsonnet/expand.libsonnet +++ b/src/risedevtool/jsonnet/expand.libsonnet @@ -1,4 +1,7 @@ // Post-process on a profile. +// TODO: expanding `provide-xx` fields is essentially to be compatible with the existing code +// to extract the information of other services inside a config. This might not be necessary +// as we can simply also pass the configs of sibling services. local pruneProvide = function(step) { diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index c132a4fedb3eb..d69b9bc668ff5 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -360,16 +360,17 @@ fn main() -> Result<()> { .arg("-V") .arg(format!("profile={task_name}")) .output() - .context("failed to evaluate risedev.jsonnet")?; + .context("`risedev-dev` requires `rsjsonnet` to be installed to expand profiles")?; if !json.status.success() { bail!( - "failed to evaluate RiseDev profile configuration:\n{}", + "failed to evaluate profile configuration:\n{}", String::from_utf8_lossy(&json.stderr) ); } let input = json.stdout; + // Dump the expanded profile to `.risingwave/risedev-profile-expanded.json` for debugging. fs_err::write( Path::new(&env::var("PREFIX_CONFIG")?).join("risedev-profile-expanded.json"), &input, diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 7fcbb17053a9a..ee4fc1ab19979 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -23,7 +23,6 @@ pub struct ComputeNodeConfig { pub id: String, pub address: String, - pub port: u16, pub listen_address: String, pub exporter_port: u16, From 221a854537571009e511eabd57d62a1e2b7001d4 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 18:22:28 +0800 Subject: [PATCH 09/10] support user profiles compat Signed-off-by: Bugen Zhao --- src/risedevtool/jsonnet/profiles-compat.libsonnet | 6 +++++- src/risedevtool/src/bin/risedev-dev.rs | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/risedevtool/jsonnet/profiles-compat.libsonnet b/src/risedevtool/jsonnet/profiles-compat.libsonnet index 035d62324b286..260c6da1d9a34 100644 --- a/src/risedevtool/jsonnet/profiles-compat.libsonnet +++ b/src/risedevtool/jsonnet/profiles-compat.libsonnet @@ -2,6 +2,10 @@ local yaml = importstr '../../../risedev.yml'; local profiles = std.parseYaml(yaml).profile; +local userYaml = importstr '../../../risedev-profiles.user.yml'; +local userProfiles = std.parseYaml(userYaml); + +local allProfiles = profiles + userProfiles; local _ = import '../../../risedev-template.libsonnet'; @@ -26,4 +30,4 @@ local mapProfile = function(name, profile) (if 'config-path' in profile then { configPath: profile['config-path'] } else {}) { steps: std.map(mapStep, profile.steps) }; -std.mapWithKey(mapProfile, profiles) +std.mapWithKey(mapProfile, allProfiles) diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index d69b9bc668ff5..bc9e81674d951 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(iterator_try_collect)] - use std::env; use std::fmt::Write; use std::path::{Path, PathBuf}; From 9ebf7a224b582ee8ef70de89fbf60f544e605357 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 9 May 2024 18:37:16 +0800 Subject: [PATCH 10/10] fix merge conflict Signed-off-by: Bugen Zhao --- src/risedevtool/src/service_config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 52c87fa448262..da9c7bd003d39 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -263,7 +263,6 @@ pub struct KafkaConfig { pub address: String, pub port: u16, - #[serde(with = "string")] pub controller_port: u16, pub listen_address: String,