feat: add 'sqlness-cli' to run sqlness in blackbox mode

.cargo/config.toml

@@ -3,7 +3,7 @@ linker = "aarch64-linux-gnu-gcc"
 
 [alias]
 sqlness = "run --bin sqlness-runner --"
-
+sqlness-cli = "run --bin sqlness-cli --"
 
 [build]
 rustflags = [

Cargo.toml

@@ -51,6 +51,8 @@ members = [
     "src/table",
     "tests-integration",
     "tests/runner",
+    "tests/cli",
+    "tests/util",
 ]
 resolver = "2"
 
@@ -165,6 +167,7 @@ script = { path = "src/script" }
 servers = { path = "src/servers" }
 session = { path = "src/session" }
 sql = { path = "src/sql" }
+sqlness-util = { path = "tests/util" }
 storage = { path = "src/storage" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }

tests/cli/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "sqlness-cli"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+async-trait = "0.1"
+clap = { version = "4.0", features = ["derive"] }
+client = { workspace = true }
+sqlness = { version = "0.5" }
+sqlness-util = { workspace = true }
+tokio.workspace = true

tests/cli/src/main.rs

@@ -0,0 +1,89 @@
+use std::fmt::Display;
+use std::path::{Path, PathBuf};
+use std::process::exit;
+
+use async_trait::async_trait;
+use clap::Parser;
+use client::{Client, Database as DB, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use sqlness::{ConfigBuilder, Database, EnvController, QueryContext, Runner};
+use tokio::sync::Mutex as TokioMutex;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+/// A cli to run sqlness tests.
+struct Args {
+    /// Directory of test cases
+    #[clap(short, long)]
+    case_dir: Option<PathBuf>,
+
+    /// Fail this run as soon as one case fails if true
+    #[arg(short, long, default_value = "false")]
+    fail_fast: bool,
+
+    /// Name of test cases to run. Accept as a regexp.
+    #[clap(short, long, default_value = ".*")]
+    test_filter: String,
+
+    /// Address of the server
+    #[clap(short, long, default_value = "127.0.0.1:4001")]
+    server_addr: String,
+}
+
+pub struct GreptimeDB {
+    client: TokioMutex<DB>,
+}
+
+#[async_trait]
+impl Database for GreptimeDB {
+    async fn query(&self, ctx: QueryContext, query: String) -> Box<dyn Display> {
+        sqlness_util::do_query(ctx, self.client.lock().await, query).await
+    }
+}
+
+pub struct CliController {
+    server_addr: String,
+}
+
+impl CliController {
+    pub fn new(server_addr: String) -> Self {
+        Self { server_addr }
+    }
+
+    async fn connect(&self) -> GreptimeDB {
+        let client = Client::with_urls(vec![self.server_addr.clone()]);
+        let db = DB::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+
+        GreptimeDB {
+            client: TokioMutex::new(db),
+        }
+    }
+}
+
+#[async_trait]
+impl EnvController for CliController {
+    type DB = GreptimeDB;
+
+    async fn start(&self, mode: &str, _config: Option<&Path>) -> Self::DB {
+        match mode {
+            "standalone" => self.connect().await,
+            "distributed" => self.connect().await,
+            _ => panic!("Unexpected mode: {mode}"),
+        }
+    }
+
+    async fn stop(&self, _mode: &str, _database: Self::DB) {}
+}
+
+#[tokio::main]
+async fn main() {
+    let args = Args::parse();
+    let config = ConfigBuilder::default()
+        .case_dir(sqlness_util::get_case_dir(args.case_dir))
+        .fail_fast(args.fail_fast)
+        .test_filter(args.test_filter)
+        .follow_links(true)
+        .build()
+        .unwrap();
+    let runner = Runner::new(config, CliController::new(args.server_addr));
+    runner.run().await.unwrap();
+}

tests/runner/Cargo.toml

@@ -8,13 +8,9 @@ license.workspace = true
 async-trait = "0.1"
 clap = { version = "4.0", features = ["derive"] }
 client = { workspace = true }
-common-base = { workspace = true }
-common-error = { workspace = true }
-common-grpc = { workspace = true }
-common-query = { workspace = true }
-common-recordbatch = { workspace = true }
 common-time = { workspace = true }
 serde.workspace = true
 sqlness = { version = "0.5" }
+sqlness-util = { workspace = true }
 tinytemplate = "1.2"
 tokio.workspace = true

tests/runner/src/env.rs

@@ -23,22 +23,16 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use async_trait::async_trait;
-use client::error::ServerSnafu;
-use client::{
-    Client, Database as DB, Error as ClientError, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME,
-};
-use common_error::ext::ErrorExt;
-use common_query::Output;
-use common_recordbatch::RecordBatches;
+use client::{Client, Database as DB, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use serde::Serialize;
 use sqlness::{Database, EnvController, QueryContext};
 use tinytemplate::TinyTemplate;
 use tokio::sync::Mutex as TokioMutex;
 
-use crate::util;
-
 const METASRV_ADDR: &str = "127.0.0.1:3002";
+
 const SERVER_ADDR: &str = "127.0.0.1:4001";
+
 const DEFAULT_LOG_LEVEL: &str = "--log-level=debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info";
 
 pub struct Env {
@@ -188,7 +182,7 @@ impl Env {
             _ => panic!("Unexpected subcommand: {subcommand}"),
         };
 
-        if util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(1)).await {
+        if sqlness_util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(1)).await {
             panic!(
                 "Port {check_ip_addr} is already in use, please check and retry.",
                 check_ip_addr = check_ip_addr
@@ -201,7 +195,7 @@ impl Env {
         let program = "greptime.exe";
 
         let mut process = Command::new(program)
-            .current_dir(util::get_binary_dir("debug"))
+            .current_dir(sqlness_util::get_binary_dir("debug"))
             .env("TZ", "UTC")
             .args(args)
             .stdout(log_file)
@@ -210,7 +204,8 @@ impl Env {
                 panic!("Failed to start the DB with subcommand {subcommand},Error: {error}")
             });
 
-        if !util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(10)).await {
+        if !sqlness_util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(10)).await
+        {
             Env::stop_server(&mut process);
             panic!("{subcommand} doesn't up in 10 seconds, quit.")
         }
@@ -315,7 +310,7 @@ impl Env {
     async fn build_db() {
         println!("Going to build the DB...");
         let output = Command::new("cargo")
-            .current_dir(util::get_workspace_root())
+            .current_dir(sqlness_util::get_workspace_root())
             .args(["build", "--bin", "greptime"])
             .output()
             .expect("Failed to start GreptimeDB");
@@ -348,35 +343,7 @@ impl Database for GreptimeDB {
             self.env.restart_server(self).await;
         }
 
-        let mut client = self.client.lock().await;
-        if query.trim().to_lowercase().starts_with("use ") {
-            let database = query
-                .split_ascii_whitespace()
-                .nth(1)
-                .expect("Illegal `USE` statement: expecting a database.")
-                .trim_end_matches(';');
-            client.set_schema(database);
-            Box::new(ResultDisplayer {
-                result: Ok(Output::AffectedRows(0)),
-            }) as _
-        } else {
-            let mut result = client.sql(&query).await;
-            if let Ok(Output::Stream(stream)) = result {
-                match RecordBatches::try_collect(stream).await {
-                    Ok(recordbatches) => result = Ok(Output::RecordBatches(recordbatches)),
-                    Err(e) => {
-                        let status_code = e.status_code();
-                        let msg = e.output_msg();
-                        result = ServerSnafu {
-                            code: status_code,
-                            msg,
-                        }
-                        .fail();
-                    }
-                }
-            }
-            Box::new(ResultDisplayer { result }) as _
-        }
+        sqlness_util::do_query(ctx, self.client.lock().await, query).await
     }
 }
 
@@ -429,38 +396,3 @@ impl GreptimeDBContext {
         self.datanode_id.store(0, Ordering::Relaxed);
     }
 }
-
-struct ResultDisplayer {
-    result: Result<Output, ClientError>,
-}
-
-impl Display for ResultDisplayer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match &self.result {
-            Ok(result) => match result {
-                Output::AffectedRows(rows) => {
-                    write!(f, "Affected Rows: {rows}")
-                }
-                Output::RecordBatches(recordbatches) => {
-                    let pretty = recordbatches.pretty_print().map_err(|e| e.to_string());
-                    match pretty {
-                        Ok(s) => write!(f, "{s}"),
-                        Err(e) => {
-                            write!(f, "Failed to pretty format {recordbatches:?}, error: {e}")
-                        }
-                    }
-                }
-                Output::Stream(_) => unreachable!(),
-            },
-            Err(e) => {
-                let status_code = e.status_code();
-                let root_cause = e.output_msg();
-                write!(
-                    f,
-                    "Error: {}({status_code}), {root_cause}",
-                    status_code as u32
-                )
-            }
-        }
-    }
-}

tests/runner/src/main.rs

@@ -19,7 +19,6 @@ use env::Env;
 use sqlness::{ConfigBuilder, Runner};
 
 mod env;
-mod util;
 
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
@@ -52,7 +51,7 @@ async fn main() {
     let data_home = std::path::PathBuf::from("/tmp");
 
     let config = ConfigBuilder::default()
-        .case_dir(util::get_case_dir(args.case_dir))
+        .case_dir(sqlness_util::get_case_dir(args.case_dir))
         .fail_fast(args.fail_fast)
         .test_filter(args.test_filter)
         .follow_links(true)

tests/util/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "sqlness-util"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+client = { workspace = true }
+common-error = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+sqlness = { version = "0.5" }
+tokio.workspace = true