test: introduce unstable fuzz create table test (#3788)

* feat: implement unstable_fuzz_create_table_standalone

* chore: use drop database

* docs: update docs

* chore: add ci config

* chore: add feature gate

* fix: fix clippy

* chore: update ci

* Apply suggestions from code review

* feat: reduce num

* Apply suggestions from code review

* chore: apply suggestions from CR

* Apply suggestions from code review

* chore: reduce `wait_timeout` in health check

* Update .env.example

* refactor: use `init_greptime_connections_via_env`

* refactor: use `init_greptime_connections_via_env`

---------

Co-authored-by: tison <[email protected]>

.env.example

@@ -24,3 +24,7 @@ GT_KAFKA_ENDPOINTS = localhost:9092
 
 # Setting for fuzz tests
 GT_MYSQL_ADDR = localhost:4002
+
+# Setting for unstable fuzz tests
+GT_FUZZ_BINARY_PATH=/path/to/
+GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/unstable_greptime

.github/actions/fuzz-test/action.yaml

@@ -3,11 +3,17 @@ description: 'Fuzz test given setup and service'
 inputs:
   target:
     description: "The fuzz target to test"
+    required: true
+  max-total-time:
+    description: "Max total time(secs)"
+    required: true
+  unstable:
+    default: 'false'
+    description: "Enable unstable feature"
 runs:
   using: composite
   steps:
   - name: Run Fuzz Test
     shell: bash
-    run: cargo fuzz run ${{ inputs.target }} --fuzz-dir tests-fuzz -D -s none -- -max_total_time=120
-    env:
-      GT_MYSQL_ADDR: 127.0.0.1:4002
+    run: cargo fuzz run ${{ inputs.target }} --fuzz-dir tests-fuzz -D -s none ${{ inputs.unstable == 'true' && '--features=unstable' || '' }} -- -max_total_time=${{ inputs.max-total-time }}
+

.github/workflows/develop.yml

@@ -171,8 +171,62 @@ jobs:
         uses: ./.github/actions/fuzz-test
         env:
           CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+          GT_MYSQL_ADDR: 127.0.0.1:4002
         with:
           target: ${{ matrix.target }}
+          max-total-time: 120
+
+  unstable-fuzztest:
+    name: Unstable Fuzz Test
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [ "unstable_fuzz_create_table_standalone" ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares across multiple jobs
+          shared-key: "fuzz-test-targets"
+      - name: Set Rust Fuzz
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install -y libfuzzer-14-dev
+          cargo install cargo-fuzz
+      - name: Download pre-built binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: bins
+          path: .
+      - name: Unzip binaries
+        run: tar -xvf ./bins.tar.gz
+      - name: Fuzz Test
+        uses: ./.github/actions/fuzz-test
+        env:
+          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+          GT_MYSQL_ADDR: 127.0.0.1:4002
+          GT_FUZZ_BINARY_PATH: ./bins/greptime
+          GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/unstable-greptime/
+        with:
+          target: ${{ matrix.target }}
+          max-total-time: 120
+          unstable: 'true'
+      - name: Upload unstable fuzz test logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: unstable-fuzz-logs
+          path: /tmp/unstable-greptime/
+          retention-days: 3
+
 
   sqlness:
     name: Sqlness Test

tests-fuzz/Cargo.toml

@@ -10,6 +10,10 @@ workspace = true
 [package.metadata]
 cargo-fuzz = true
 
+[features]
+default = []
+unstable = ["nix"]
+
 [dependencies]
 arbitrary = { version = "1.3.0", features = ["derive"] }
 async-trait = { workspace = true }
@@ -24,9 +28,11 @@ derive_builder = { workspace = true }
 dotenv = "0.15"
 lazy_static = { workspace = true }
 libfuzzer-sys = "0.4"
+nix = { version = "0.28", features = ["process", "signal"], optional = true }
 partition = { workspace = true }
 rand = { workspace = true }
 rand_chacha = "0.3.1"
+reqwest = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 snafu = { workspace = true }
@@ -38,10 +44,11 @@ sqlx = { version = "0.6", features = [
     "postgres",
     "chrono",
 ] }
+tinytemplate = "1.2"
+tokio = { workspace = true }
 
 [dev-dependencies]
 dotenv.workspace = true
-tokio = { workspace = true }
 
 [[bin]]
 name = "fuzz_create_table"
@@ -91,3 +98,11 @@ path = "targets/fuzz_create_database.rs"
 test = false
 bench = false
 doc = false
+
+[[bin]]
+name = "unstable_fuzz_create_table_standalone"
+path = "targets/unstable/fuzz_create_table_standalone.rs"
+test = false
+bench = false
+doc = false
+required-features = ["unstable"]

tests-fuzz/README.md

@@ -9,6 +9,22 @@ cargo install cargo-fuzz
 2. Start GreptimeDB
 3. Copy the `.env.example`, which is at project root, to `.env` and change the values on need.
 
+### For stable fuzz tests
+Set the GreptimeDB MySQL address.
+```
+GT_MYSQL_ADDR = localhost:4002
+```
+
+### For unstable fuzz tests
+Set the binary path of the GreptimeDB:
+```
+GT_FUZZ_BINARY_PATH = /path/to/
+```
+
+Change the instance root directory(the default value: `/tmp/unstable_greptime/`)
+```
+GT_FUZZ_INSTANCE_ROOT_DIR = /path/to/
+```
 ## Run
 1. List all fuzz targets
 ```bash

tests-fuzz/conf/standalone.template.toml

@@ -0,0 +1,23 @@
+mode = 'standalone'
+enable_memory_catalog = false
+require_lease_before_startup = true
+
+[wal]
+provider = "raft_engine"
+file_size = '1GB'
+purge_interval = '10m'
+purge_threshold = '10GB'
+read_batch_size = 128
+sync_write = false
+
+[storage]
+type = 'File'
+data_home = '{data_home}'
+
+[grpc_options]
+addr = '127.0.0.1:4001'
+runtime_size = 8
+
+[procedure]
+max_retry_times = 3
+retry_delay = "500ms"

tests-fuzz/src/error.rs

@@ -16,13 +16,31 @@ use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 use crate::ir::create_expr::{CreateDatabaseExprBuilderError, CreateTableExprBuilderError};
+#[cfg(feature = "unstable")]
+use crate::utils::process::Pid;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Snafu)]
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("Failed to create a file: {}", path))]
+    CreateFile {
+        path: String,
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Failed to write a file: {}", path))]
+    WriteFile {
+        path: String,
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
     #[snafu(display("Unexpected, violated: {violated}"))]
     Unexpected {
         violated: String,
@@ -56,4 +74,23 @@ pub enum Error {
 
     #[snafu(display("Failed to assert: {}", reason))]
     Assert { reason: String, location: Location },
+
+    #[snafu(display("Child process exited unexpected"))]
+    UnexpectedExited { location: Location },
+
+    #[snafu(display("Failed to spawn a child process"))]
+    SpawnChild {
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[cfg(feature = "unstable")]
+    #[snafu(display("Failed to kill a process, pid: {}", pid))]
+    KillProcess {
+        location: Location,
+        #[snafu(source)]
+        error: nix::Error,
+        pid: Pid,
+    },
 }

tests-fuzz/src/utils.rs

@@ -12,31 +12,74 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod config;
+pub mod health;
+#[cfg(feature = "unstable")]
+pub mod process;
+
 use std::env;
 
 use common_telemetry::info;
 use sqlx::mysql::MySqlPoolOptions;
 use sqlx::{MySql, Pool};
 
+/// Database connections
 pub struct Connections {
     pub mysql: Option<Pool<MySql>>,
 }
 
 const GT_MYSQL_ADDR: &str = "GT_MYSQL_ADDR";
 
-pub async fn init_greptime_connections() -> Connections {
+/// Connects to GreptimeDB via env variables.
+pub async fn init_greptime_connections_via_env() -> Connections {
     let _ = dotenv::dotenv();
     let mysql = if let Ok(addr) = env::var(GT_MYSQL_ADDR) {
+        Some(addr)
+    } else {
+        info!("GT_MYSQL_ADDR is empty, ignores test");
+        None
+    };
+
+    init_greptime_connections(mysql).await
+}
+
+/// Connects to GreptimeDB.
+pub async fn init_greptime_connections(mysql: Option<String>) -> Connections {
+    let mysql = if let Some(addr) = mysql {
         Some(
             MySqlPoolOptions::new()
                 .connect(&format!("mysql://{addr}/public"))
                 .await
                 .unwrap(),
         )
     } else {
-        info!("GT_MYSQL_ADDR is empty, ignores test");
         None
     };
 
     Connections { mysql }
 }
+
+const GT_FUZZ_BINARY_PATH: &str = "GT_FUZZ_BINARY_PATH";
+const GT_FUZZ_INSTANCE_ROOT_DIR: &str = "GT_FUZZ_INSTANCE_ROOT_DIR";
+
+/// The variables for unstable test
+pub struct UnstableTestVariables {
+    pub binary_path: String,
+    pub root_dir: Option<String>,
+}
+
+/// Loads env variables for unstable test
+pub fn load_unstable_test_env_variables() -> UnstableTestVariables {
+    let _ = dotenv::dotenv();
+    let binary_path = env::var(GT_FUZZ_BINARY_PATH).expect("GT_FUZZ_BINARY_PATH not found");
+    let root_dir = if let Ok(root) = env::var(GT_FUZZ_INSTANCE_ROOT_DIR) {
+        Some(root)
+    } else {
+        None
+    };
+
+    UnstableTestVariables {
+        binary_path,
+        root_dir,
+    }
+}

tests-fuzz/src/utils/config.rs

@@ -0,0 +1,58 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+
+use common_telemetry::tracing::info;
+use serde::Serialize;
+use snafu::ResultExt;
+use tinytemplate::TinyTemplate;
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
+
+use crate::error;
+use crate::error::Result;
+
+/// Get the path of config dir `tests-fuzz/conf`.
+pub fn get_conf_path() -> PathBuf {
+    let mut root_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    root_path.push("conf");
+    root_path
+}
+
+/// Returns rendered config file.
+pub fn render_config_file<C: Serialize>(template_path: &str, context: &C) -> String {
+    let mut tt = TinyTemplate::new();
+    let template = std::fs::read_to_string(template_path).unwrap();
+    tt.add_template(template_path, &template).unwrap();
+    tt.render(template_path, context).unwrap()
+}
+
+// Writes config file to `output_path`.
+pub async fn write_config_file<C: Serialize>(
+    template_path: &str,
+    context: &C,
+    output_path: &str,
+) -> Result<()> {
+    info!("template_path: {template_path}, output_path: {output_path}");
+    let content = render_config_file(template_path, context);
+    let mut config_file = File::create(output_path)
+        .await
+        .context(error::CreateFileSnafu { path: output_path })?;
+    config_file
+        .write_all(content.as_bytes())
+        .await
+        .context(error::WriteFileSnafu { path: output_path })?;
+    Ok(())
+}