From d25c1b5bd5c2f6c7d23226a2652e6a1e4173d008 Mon Sep 17 00:00:00 2001 From: August Date: Tue, 14 May 2024 15:30:13 +0800 Subject: [PATCH] feat: support pg service & pg meta backend in risedev using docker (#16662) --- risedev.yml | 98 +++++++++++++++++++ src/cmd_all/src/single_node.rs | 2 +- src/meta/node/src/lib.rs | 8 +- src/risedevtool/src/bin/risedev-compose.rs | 2 +- src/risedevtool/src/bin/risedev-dev.rs | 16 ++- src/risedevtool/src/config.rs | 1 + src/risedevtool/src/risedev_env.rs | 16 +++ src/risedevtool/src/service_config.rs | 26 +++++ src/risedevtool/src/task.rs | 2 + src/risedevtool/src/task/meta_node_service.rs | 81 +++++++++------ src/risedevtool/src/task/postgres_service.rs | 55 +++++++++++ 11 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 src/risedevtool/src/task/postgres_service.rs diff --git a/risedev.yml b/risedev.yml index ec057c5bf2c4c..a6ce8c691d80a 100644 --- a/risedev.yml +++ b/risedev.yml @@ -108,6 +108,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -124,6 +125,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -143,6 +145,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -156,6 +159,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -260,6 +264,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -295,6 +300,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: etcd - use: meta-node port: 15690 dashboard-port: 15691 @@ -342,6 +348,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: etcd - use: meta-node port: 15690 dashboard-port: 15691 @@ -362,6 +369,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: sqlite - use: compactor - use: compute-node - use: frontend @@ -375,6 +383,40 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: sqlite + - use: compactor + - use: compute-node + - use: frontend + + meta-1cn-1fe-pg-backend: + steps: + - use: minio + - use: postgres + port: 8432 + user: postgres + database: metadata + - use: meta-node + port: 5690 + dashboard-port: 5691 + exporter-port: 1250 + meta-backend: postgres + - use: compactor + - use: compute-node + - use: frontend + + meta-1cn-1fe-pg-backend-with-recovery: + config-path: src/config/ci-recovery.toml + steps: + - use: minio + - use: postgres + port: 8432 + user: postgres + database: metadata + - use: meta-node + port: 5690 + dashboard-port: 5691 + exporter-port: 1250 + meta-backend: postgres - use: compactor - use: compute-node - use: frontend @@ -401,6 +443,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node parallelism: 8 - use: frontend @@ -561,6 +604,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -573,6 +617,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -595,6 +640,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -621,6 +667,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -645,6 +692,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -669,6 +717,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -728,6 +777,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: opendal engine: fs bucket: "/tmp/rw_ci" @@ -761,6 +811,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -812,6 +863,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -830,6 +882,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -854,6 +907,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -867,6 +921,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true total-memory-bytes: 17179869184 @@ -879,6 +934,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -891,6 +947,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -912,6 +969,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -927,6 +985,7 @@ profile: - use: etcd - use: minio - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -943,6 +1002,7 @@ profile: - use: sqlite - use: minio - use: meta-node + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor @@ -993,6 +1053,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -1004,6 +1065,7 @@ profile: bucket: renjie-iceberg-bench - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -1016,6 +1078,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -1176,12 +1239,18 @@ template: # If `user-managed` is true, this service will be started by user with the above config user-managed: false + # meta backend type, requires extra config for provided backend + meta-backend: "memory" + # Etcd backend config provide-etcd-backend: "etcd*" # Sqlite backend config provide-sqlite-backend: "sqlite*" + # Postgres backend config + provide-postgres-backend: "postgres*" + # Prometheus nodes used by dashboard service provide-prometheus: "prometheus*" @@ -1476,3 +1545,32 @@ template: # If `user-managed` is true, user is responsible for starting the service # to serve at the above address and port in any way they see fit. user-managed: false + + # PostgreSQL service backed by docker. + postgres: + # Id to be picked-up by services + id: postgres-${port} + + # address of pg + address: "127.0.0.1" + + # listen port of pg + port: 8432 + + # Note: + # - This will be used to initialize the PostgreSQL instance if it's fresh. + # - In user-managed mode, these configs are not validated by risedev. + # They are passed as-is to risedev-env default user for PostgreSQL operations. + user: postgres + password: "" + database: "postgres" + + # The docker image. Can be overridden to use a different version. + image: "postgres:15-alpine" + + # If set to true, data will be persisted at data/{id}. + persist-data: true + + # If `user-managed` is true, user is responsible for starting the service + # to serve at the above address and port in any way they see fit. + user-managed: false diff --git a/src/cmd_all/src/single_node.rs b/src/cmd_all/src/single_node.rs index 63099c1ff640c..2340841a13c31 100644 --- a/src/cmd_all/src/single_node.rs +++ b/src/cmd_all/src/single_node.rs @@ -186,7 +186,7 @@ pub fn map_single_node_opts_to_standalone_opts(opts: SingleNodeOpts) -> ParsedSt std::fs::create_dir_all(&meta_store_dir).unwrap(); let meta_store_endpoint = format!("sqlite://{}/single_node.db?mode=rwc", &meta_store_dir); - meta_opts.sql_endpoint = Some(meta_store_endpoint); + meta_opts.sql_endpoint = Some(meta_store_endpoint.into()); } } diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs index 140cf481e3c40..c7d8e95d7e49d 100644 --- a/src/meta/node/src/lib.rs +++ b/src/meta/node/src/lib.rs @@ -76,7 +76,7 @@ pub struct MetaNodeOpts { /// Endpoint of the SQL service, make it non-option when SQL service is required. #[clap(long, hide = true, env = "RW_SQL_ENDPOINT")] - pub sql_endpoint: Option, + pub sql_endpoint: Option>, /// The HTTP REST-API address of the Prometheus instance associated to this cluster. /// This address is used to serve `PromQL` queries to Prometheus. @@ -221,7 +221,11 @@ pub fn start(opts: MetaNodeOpts) -> Pin + Send>> { }, MetaBackend::Mem => MetaStoreBackend::Mem, MetaBackend::Sql => MetaStoreBackend::Sql { - endpoint: opts.sql_endpoint.expect("sql endpoint is required"), + endpoint: opts + .sql_endpoint + .expect("sql endpoint is required") + .expose_secret() + .to_string(), }, }; diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index c3a7d079e4c4e..2c88311ccca5d 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -222,7 +222,7 @@ fn main() -> Result<()> { volumes.insert(c.id.clone(), ComposeVolume::default()); (c.address.clone(), c.compose(&compose_config)?) } - ServiceConfig::Redis(_) | ServiceConfig::MySql(_) => { + ServiceConfig::Redis(_) | ServiceConfig::MySql(_) | ServiceConfig::Postgres(_) => { return Err(anyhow!("not supported")) } }; diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index 3c5ad8aa082e0..d8307a542bc6d 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -26,9 +26,9 @@ use risedev::util::{complete_spin, fail_spin}; use risedev::{ generate_risedev_env, preflight_check, CompactorService, ComputeNodeService, ConfigExpander, ConfigureTmuxTask, DummyService, EnsureStopService, ExecuteContext, FrontendService, - GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PrometheusService, - PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, TempoService, ZooKeeperService, - RISEDEV_NAME, + GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PostgresService, + PrometheusService, PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, + TempoService, ZooKeeperService, RISEDEV_NAME, }; use tempfile::tempdir; use thiserror_ext::AsReport; @@ -326,6 +326,16 @@ fn task_main( ctx.pb .set_message(format!("mysql {}:{}", c.address, c.port)); } + ServiceConfig::Postgres(c) => { + let mut ctx = + ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone()); + PostgresService::new(c.clone()).execute(&mut ctx)?; + let mut task = + risedev::ConfigureTcpNodeTask::new(c.address.clone(), c.port, c.user_managed)?; + task.execute(&mut ctx)?; + ctx.pb + .set_message(format!("postgres {}:{}", c.address, c.port)); + } } let service_id = service.id().to_string(); diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs index 45d90daa0c872..239a822725340 100644 --- a/src/risedevtool/src/config.rs +++ b/src/risedevtool/src/config.rs @@ -175,6 +175,7 @@ impl ConfigExpander { "zookeeper" => ServiceConfig::ZooKeeper(serde_yaml::from_str(&out_str)?), "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?), "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?), + "postgres" => ServiceConfig::Postgres(serde_yaml::from_str(&out_str)?), other => return Err(anyhow!("unsupported use type: {}", other)), }; Ok(result) diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs index 24f18895434e9..b33a65f9986aa 100644 --- a/src/risedevtool/src/risedev_env.rs +++ b/src/risedevtool/src/risedev_env.rs @@ -91,6 +91,22 @@ pub fn generate_risedev_env(services: &Vec) -> String { // It's expected to create another dedicated user for the source. writeln!(env, r#"RISEDEV_MYSQL_WITH_OPTIONS_COMMON="connector='mysql-cdc',hostname='{host}',port='{port}'""#,).unwrap(); } + ServiceConfig::Pubsub(c) => { + let address = &c.address; + let port = &c.port; + writeln!(env, r#"RISEDEV_PUBSUB_WITH_OPTIONS_COMMON="connector='google_pubsub',pubsub.emulator_host='{address}:{port}'""#,).unwrap(); + } + ServiceConfig::Postgres(c) => { + let host = &c.address; + let port = &c.port; + let user = &c.user; + let password = &c.password; + // These envs are used by `postgres` cli. + writeln!(env, r#"PGHOST="{host}""#,).unwrap(); + writeln!(env, r#"PGPORT="{port}""#,).unwrap(); + writeln!(env, r#"PGUSER="{user}""#,).unwrap(); + writeln!(env, r#"PGPASSWORD="{password}""#,).unwrap(); + } _ => {} } } diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 27231552dfc2d..73b0fdd585c5f 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -60,8 +60,10 @@ pub struct MetaNodeConfig { pub user_managed: bool, + pub meta_backend: String, pub provide_etcd_backend: Option>, pub provide_sqlite_backend: Option>, + pub provide_postgres_backend: Option>, pub provide_prometheus: Option>, pub provide_compute_node: Option>, @@ -356,6 +358,26 @@ pub struct MySqlConfig { pub persist_data: bool, } +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] +pub struct PostgresConfig { + #[serde(rename = "use")] + phantom_use: Option, + pub id: String, + + pub port: u16, + pub address: String, + + pub user: String, + pub password: String, + pub database: String, + + pub image: String, + pub user_managed: bool, + pub persist_data: bool, +} + /// All service configuration #[derive(Clone, Debug, PartialEq)] pub enum ServiceConfig { @@ -377,6 +399,7 @@ pub enum ServiceConfig { ZooKeeper(ZooKeeperConfig), RedPanda(RedPandaConfig), MySql(MySqlConfig), + Postgres(PostgresConfig), } impl ServiceConfig { @@ -400,6 +423,7 @@ impl ServiceConfig { Self::RedPanda(c) => &c.id, Self::Opendal(c) => &c.id, Self::MySql(c) => &c.id, + ServiceConfig::Postgres(c) => &c.id, } } @@ -423,6 +447,7 @@ impl ServiceConfig { Self::RedPanda(_c) => None, Self::Opendal(_) => None, Self::MySql(c) => Some(c.port), + ServiceConfig::Postgres(c) => Some(c.port), } } @@ -446,6 +471,7 @@ impl ServiceConfig { Self::RedPanda(_c) => false, Self::Opendal(_c) => false, Self::MySql(c) => c.user_managed, + Self::Postgres(c) => c.user_managed, } } } diff --git a/src/risedevtool/src/task.rs b/src/risedevtool/src/task.rs index a0ce29c4d2ba6..97d9fc75b52c9 100644 --- a/src/risedevtool/src/task.rs +++ b/src/risedevtool/src/task.rs @@ -25,6 +25,7 @@ mod kafka_service; mod meta_node_service; mod minio_service; mod mysql_service; +mod postgres_service; mod prometheus_service; mod pubsub_service; mod redis_service; @@ -63,6 +64,7 @@ pub use self::kafka_service::*; pub use self::meta_node_service::*; pub use self::minio_service::*; pub use self::mysql_service::*; +pub use self::postgres_service::*; pub use self::prometheus_service::*; pub use self::pubsub_service::*; pub use self::redis_service::*; diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs index 42d555f48b3d8..a1bbd2c04e6ca 100644 --- a/src/risedevtool/src/task/meta_node_service.rs +++ b/src/risedevtool/src/task/meta_node_service.rs @@ -77,35 +77,58 @@ impl MetaNodeService { let mut is_persistent_meta_store = false; - if let Some(sqlite_config) = &config.provide_sqlite_backend - && !sqlite_config.is_empty() - { - is_persistent_meta_store = true; - let prefix_data = env::var("PREFIX_DATA")?; - let file_path = PathBuf::from(&prefix_data) - .join(&sqlite_config[0].id) - .join(&sqlite_config[0].file); - cmd.arg("--backend") - .arg("sql") - .arg("--sql-endpoint") - .arg(format!("sqlite://{}?mode=rwc", file_path.display())); - } else { - match config.provide_etcd_backend.as_ref().unwrap().as_slice() { - [] => { - cmd.arg("--backend").arg("mem"); - } - etcds => { - is_persistent_meta_store = true; - cmd.arg("--backend") - .arg("etcd") - .arg("--etcd-endpoints") - .arg( - etcds - .iter() - .map(|etcd| format!("{}:{}", etcd.address, etcd.port)) - .join(","), - ); - } + match config.meta_backend.to_ascii_lowercase().as_str() { + "memory" => { + cmd.arg("--backend").arg("mem"); + } + "etcd" => { + let etcd_config = config.provide_etcd_backend.as_ref().unwrap(); + assert!(!etcd_config.is_empty()); + is_persistent_meta_store = true; + + cmd.arg("--backend") + .arg("etcd") + .arg("--etcd-endpoints") + .arg( + etcd_config + .iter() + .map(|etcd| format!("{}:{}", etcd.address, etcd.port)) + .join(","), + ); + } + "sqlite" => { + let sqlite_config = config.provide_sqlite_backend.as_ref().unwrap(); + assert_eq!(sqlite_config.len(), 1); + is_persistent_meta_store = true; + + let prefix_data = env::var("PREFIX_DATA")?; + let file_path = PathBuf::from(&prefix_data) + .join(&sqlite_config[0].id) + .join(&sqlite_config[0].file); + cmd.arg("--backend") + .arg("sql") + .arg("--sql-endpoint") + .arg(format!("sqlite://{}?mode=rwc", file_path.display())); + } + "postgres" => { + let pg_config = config.provide_postgres_backend.as_ref().unwrap(); + assert_eq!(pg_config.len(), 1); + is_persistent_meta_store = true; + + cmd.arg("--backend") + .arg("sql") + .arg("--sql-endpoint") + .arg(format!( + "postgres://{}:{}@{}:{}/{}", + pg_config[0].user, + pg_config[0].password, + pg_config[0].address, + pg_config[0].port, + pg_config[0].database + )); + } + backend => { + return Err(anyhow!("unsupported meta backend {}", backend)); } } diff --git a/src/risedevtool/src/task/postgres_service.rs b/src/risedevtool/src/task/postgres_service.rs new file mode 100644 index 0000000000000..4a6d6571c82cc --- /dev/null +++ b/src/risedevtool/src/task/postgres_service.rs @@ -0,0 +1,55 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::task::docker_service::{DockerService, DockerServiceConfig}; +use crate::PostgresConfig; + +impl DockerServiceConfig for PostgresConfig { + fn id(&self) -> String { + self.id.clone() + } + + fn is_user_managed(&self) -> bool { + self.user_managed + } + + fn image(&self) -> String { + self.image.clone() + } + + fn envs(&self) -> Vec<(String, String)> { + vec![ + ("POSTGRES_HOST_AUTH_METHOD".to_owned(), "trust".to_owned()), + ("POSTGRES_USER".to_owned(), self.user.clone()), + ("POSTGRES_PASSWORD".to_owned(), self.password.clone()), + ("POSTGRES_DB".to_owned(), self.database.clone()), + ( + "POSTGRES_INITDB_ARGS".to_owned(), + "--encoding=UTF-8 --lc-collate=C --lc-ctype=C".to_owned(), + ), + ] + } + + fn ports(&self) -> Vec<(String, String)> { + vec![(format!("{}:{}", self.address, self.port), "5432".to_owned())] + } + + fn data_path(&self) -> Option { + self.persist_data + .then(|| "/var/lib/postgresql/data".to_owned()) + } +} + +/// Docker-backed PostgreSQL service. +pub type PostgresService = DockerService;