feat: support config sql metastore using username and password by env (…

…#17530) (#17737)

Co-authored-by: August <[email protected]>

src/cmd_all/src/single_node.rs

@@ -173,19 +173,21 @@ pub fn map_single_node_opts_to_standalone_opts(opts: SingleNodeOpts) -> ParsedSt
     // Set meta store for meta (if not set). It could be set by environment variables before this.
     let meta_backend_is_set = match meta_opts.backend {
         Some(MetaBackend::Etcd) => !meta_opts.etcd_endpoints.is_empty(),
-        Some(MetaBackend::Sql) => meta_opts.sql_endpoint.is_some(),
+        Some(MetaBackend::Sql)
+        | Some(MetaBackend::Sqlite)
+        | Some(MetaBackend::Postgres)
+        | Some(MetaBackend::Mysql) => meta_opts.sql_endpoint.is_some(),
         Some(MetaBackend::Mem) => true,
         None => false,
     };
     if !meta_backend_is_set {
         if opts.in_memory {
             meta_opts.backend = Some(MetaBackend::Mem);
         } else {
-            meta_opts.backend = Some(MetaBackend::Sql);
+            meta_opts.backend = Some(MetaBackend::Sqlite);
             let meta_store_dir = format!("{}/meta_store", &store_directory);
             std::fs::create_dir_all(&meta_store_dir).unwrap();
-            let meta_store_endpoint =
-                format!("sqlite://{}/single_node.db?mode=rwc", &meta_store_dir);
+            let meta_store_endpoint = format!("{}/single_node.db", &meta_store_dir);
             meta_opts.sql_endpoint = Some(meta_store_endpoint.into());
         }
     }

src/cmd_all/src/standalone.rs

@@ -323,6 +323,9 @@ mod test {
                             etcd_username: "",
                             etcd_password: [REDACTED alloc::string::String],
                             sql_endpoint: None,
+                            sql_username: "",
+                            sql_password: [REDACTED alloc::string::String],
+                            sql_database: "",
                             prometheus_endpoint: None,
                             prometheus_selector: None,
                             privatelink_endpoint_default_tags: None,

src/common/src/config.rs

@@ -175,7 +175,10 @@ pub enum MetaBackend {
     #[default]
     Mem,
     Etcd,
-    Sql,
+    Sql, // keep for backward compatibility
+    Sqlite,
+    Postgres,
+    Mysql,
 }
 
 /// The section `[meta]` in `risingwave.toml`.

src/ctl/src/lib.rs

@@ -448,6 +448,7 @@ enum ScaleCommands {
 }
 
 #[derive(Subcommand)]
+#[allow(clippy::large_enum_variant)]
 enum MetaCommands {
     /// pause the stream graph
     Pause,

src/meta/node/src/lib.rs

@@ -78,6 +78,18 @@ pub struct MetaNodeOpts {
     #[clap(long, hide = true, env = "RW_SQL_ENDPOINT")]
     pub sql_endpoint: Option<Secret<String>>,
 
+    /// Username of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, hide = true, env = "RW_SQL_USERNAME", default_value = "")]
+    pub sql_username: String,
+
+    /// Password of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, hide = true, env = "RW_SQL_PASSWORD", default_value = "")]
+    pub sql_password: Secret<String>,
+
+    /// Database of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, hide = true, env = "RW_SQL_DATABASE", default_value = "")]
+    pub sql_database: String,
+
     /// The HTTP REST-API address of the Prometheus instance associated to this cluster.
     /// This address is used to serve `PromQL` queries to Prometheus.
     /// It is also used by Grafana Dashboard Service to fetch metrics and visualize them.
@@ -227,6 +239,36 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                     .expose_secret()
                     .to_string(),
             },
+            MetaBackend::Sqlite => MetaStoreBackend::Sql {
+                endpoint: format!(
+                    "sqlite://{}?mode=rwc",
+                    opts.sql_endpoint
+                        .expect("sql endpoint is required")
+                        .expose_secret()
+                ),
+            },
+            MetaBackend::Postgres => MetaStoreBackend::Sql {
+                endpoint: format!(
+                    "postgres://{}:{}@{}/{}",
+                    opts.sql_username,
+                    opts.sql_password.expose_secret(),
+                    opts.sql_endpoint
+                        .expect("sql endpoint is required")
+                        .expose_secret(),
+                    opts.sql_database
+                ),
+            },
+            MetaBackend::Mysql => MetaStoreBackend::Sql {
+                endpoint: format!(
+                    "mysql://{}:{}@{}/{}",
+                    opts.sql_username,
+                    opts.sql_password.expose_secret(),
+                    opts.sql_endpoint
+                        .expect("sql endpoint is required")
+                        .expose_secret(),
+                    opts.sql_database
+                ),
+            },
         };
 
         validate_config(&config);

src/meta/src/backup_restore/restore.rs

@@ -43,6 +43,15 @@ pub struct RestoreOpts {
     pub meta_store_type: MetaBackend,
     #[clap(long, default_value_t = String::from(""))]
     pub sql_endpoint: String,
+    /// Username of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, default_value = "")]
+    pub sql_username: String,
+    /// Password of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, default_value = "")]
+    pub sql_password: String,
+    /// Database of sql backend, required when meta backend set to MySQL or PostgreSQL.
+    #[clap(long, default_value = "")]
+    pub sql_database: String,
     /// Etcd endpoints.
     #[clap(long, default_value_t = String::from(""))]
     pub etcd_endpoints: String,
@@ -230,6 +239,9 @@ mod tests {
             meta_snapshot_id: 1,
             meta_store_type: MetaBackend::Mem,
             sql_endpoint: "".to_string(),
+            sql_username: "".to_string(),
+            sql_password: "".to_string(),
+            sql_database: "".to_string(),
             etcd_endpoints: "".to_string(),
             etcd_auth: false,
             etcd_username: "".to_string(),

src/meta/src/backup_restore/utils.rs

@@ -65,6 +65,21 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<MetaStoreBackendI
         MetaBackend::Sql => MetaStoreBackend::Sql {
             endpoint: opts.sql_endpoint,
         },
+        MetaBackend::Sqlite => MetaStoreBackend::Sql {
+            endpoint: format!("sqlite://{}?mode=rwc", opts.sql_endpoint),
+        },
+        MetaBackend::Postgres => MetaStoreBackend::Sql {
+            endpoint: format!(
+                "postgres://{}:{}@{}/{}",
+                opts.sql_username, opts.sql_password, opts.sql_endpoint, opts.sql_database
+            ),
+        },
+        MetaBackend::Mysql => MetaStoreBackend::Sql {
+            endpoint: format!(
+                "mysql://{}:{}@{}/{}",
+                opts.sql_username, opts.sql_password, opts.sql_endpoint, opts.sql_database
+            ),
+        },
     };
     match meta_store_backend {
         MetaStoreBackend::Etcd {

src/meta/src/telemetry.rs

@@ -94,7 +94,10 @@ impl TelemetryToProtobuf for MetaTelemetryReport {
             meta_backend: match self.meta_backend {
                 MetaBackend::Etcd => risingwave_pb::telemetry::MetaBackend::Etcd as i32,
                 MetaBackend::Mem => risingwave_pb::telemetry::MetaBackend::Memory as i32,
-                MetaBackend::Sql => risingwave_pb::telemetry::MetaBackend::Rdb as i32,
+                MetaBackend::Sql
+                | MetaBackend::Sqlite
+                | MetaBackend::Postgres
+                | MetaBackend::Mysql => risingwave_pb::telemetry::MetaBackend::Rdb as i32,
             },
             node_count: Some(risingwave_pb::telemetry::NodeCount {
                 meta: self.node_count.meta_count as u32,

src/risedevtool/src/task/meta_node_service.rs

@@ -109,9 +109,9 @@ impl MetaNodeService {
                     .join(&sqlite_config[0].id)
                     .join(&sqlite_config[0].file);
                 cmd.arg("--backend")
-                    .arg("sql")
+                    .arg("sqlite")
                     .arg("--sql-endpoint")
-                    .arg(format!("sqlite://{}?mode=rwc", file_path.display()));
+                    .arg(file_path);
             }
             MetaBackend::Postgres => {
                 let pg_config = config.provide_postgres_backend.as_ref().unwrap();
@@ -123,16 +123,18 @@ impl MetaNodeService {
                 is_persistent_meta_store = true;
 
                 cmd.arg("--backend")
-                    .arg("sql")
+                    .arg("postgres")
                     .arg("--sql-endpoint")
                     .arg(format!(
-                        "postgres://{}:{}@{}:{}/{}",
-                        pg_store_config.user,
-                        pg_store_config.password,
-                        pg_store_config.address,
-                        pg_store_config.port,
-                        pg_store_config.database
-                    ));
+                        "{}:{}",
+                        pg_store_config.address, pg_store_config.port,
+                    ))
+                    .arg("--sql-username")
+                    .arg(&pg_store_config.user)
+                    .arg("--sql-password")
+                    .arg(&pg_store_config.password)
+                    .arg("--sql-database")
+                    .arg(&pg_store_config.database);
             }
             MetaBackend::Mysql => {
                 let mysql_config = config.provide_mysql_backend.as_ref().unwrap();
@@ -144,16 +146,18 @@ impl MetaNodeService {
                 is_persistent_meta_store = true;
 
                 cmd.arg("--backend")
-                    .arg("sql")
+                    .arg("mysql")
                     .arg("--sql-endpoint")
                     .arg(format!(
-                        "mysql://{}:{}@{}:{}/{}",
-                        mysql_store_config.user,
-                        mysql_store_config.password,
-                        mysql_store_config.address,
-                        mysql_store_config.port,
-                        mysql_store_config.database
-                    ));
+                        "{}:{}",
+                        mysql_store_config.address, mysql_store_config.port,
+                    ))
+                    .arg("--sql-username")
+                    .arg(&mysql_store_config.user)
+                    .arg("--sql-password")
+                    .arg(&mysql_store_config.password)
+                    .arg("--sql-database")
+                    .arg(&mysql_store_config.database);
             }
         }