sql: add schema goldentests

src/encoding/format.rs

@@ -3,6 +3,7 @@
 
 use super::{bincode, Key as _};
 use crate::raft;
+use crate::sql;
 use crate::storage::mvcc;
 
 use itertools::Itertools as _;
@@ -148,3 +149,40 @@ impl<I: Formatter> Formatter for MVCC<I> {
         }
     }
 }
+
+/// Formats SQL keys/values.
+/// TODO: consider more terse formatting, e.g. dropping the value type names and
+/// instead relying on unambiguous string formatting.
+pub struct SQL;
+
+impl Formatter for SQL {
+    fn key(key: &[u8]) -> String {
+        let Ok(key) = sql::engine::Key::decode(key) else { return Raw::key(key) };
+        format!("sql:{key:?}")
+    }
+
+    fn value(key: &[u8], value: &[u8]) -> String {
+        let Ok(key) = sql::engine::Key::decode(key) else { return Raw::key(value) };
+        match key {
+            sql::engine::Key::Table(_) => {
+                let Ok(table) = bincode::deserialize::<sql::types::Table>(value) else {
+                    return Raw::bytes(value);
+                };
+                let re = regex::Regex::new(r#"\n\s*"#).expect("regex failed");
+                re.replace_all(&format!("{table}"), " ").into_owned()
+            }
+            sql::engine::Key::Row(_, _) => {
+                let Ok(row) = bincode::deserialize::<sql::types::Row>(value) else {
+                    return Raw::bytes(value);
+                };
+                row.into_iter().map(|v| format!("{v:?}")).join(",")
+            }
+            sql::engine::Key::Index(_, _, _) => {
+                let Ok(index) = bincode::deserialize::<BTreeSet<sql::types::Value>>(value) else {
+                    return Raw::bytes(value);
+                };
+                index.into_iter().map(|v| format!("{v:?}")).join(",")
+            }
+        }
+    }
+}

src/sql/engine/local.rs

@@ -14,7 +14,7 @@ use std::collections::{BTreeSet, HashMap};
 /// node-local SQL storage.
 pub struct Local<E: storage::Engine + 'static> {
     /// The local MVCC storage engine.
-    pub(super) mvcc: mvcc::MVCC<E>,
+    pub(crate) mvcc: mvcc::MVCC<E>,
 }
 
 impl<E: storage::Engine> Local<E> {
@@ -410,8 +410,8 @@ impl<E: storage::Engine> Catalog for Transaction<E> {
 /// table/column names, so this is fine.
 ///
 /// Uses Cow to allow encoding borrowed values but decoding owned values.
-#[derive(Deserialize, Serialize)]
-enum Key<'a> {
+#[derive(Debug, Deserialize, Serialize)]
+pub enum Key<'a> {
     /// A table schema by table name.
     Table(Cow<'a, str>),
     /// An index entry, by table name, index name, and index value.

src/sql/engine/mod.rs

@@ -4,6 +4,6 @@ mod raft;
 mod session;
 
 pub use engine::{Catalog, Engine, IndexScan, Transaction};
-pub use local::Local;
+pub use local::{Key, Local};
 pub use raft::{Raft, Status};
 pub use session::{Session, StatementResult};

src/sql/mod.rs

@@ -6,24 +6,160 @@ pub mod types;
 
 #[cfg(test)]
 mod tests {
+    use crate::encoding::format::{self, Formatter as _};
     use crate::sql::engine::{Engine, Local};
     use crate::sql::planner::{Planner, Scope};
     use crate::sql::types::Value;
     use crate::storage;
+    use crate::storage::engine::test::{Emit, Mirror, Operation};
+    use crate::storage::Engine as _;
+    use crossbeam::channel::Receiver;
+    use itertools::Itertools as _;
     use std::error::Error;
     use std::fmt::Write as _;
     use std::result::Result;
     use test_each_file::test_each_path;
 
+    use super::engine::{Catalog as _, Session};
     use super::parser::Parser;
 
     // Run goldenscript tests in src/sql/testscripts.
-    test_each_path! { in "src/sql/testscripts/expressions" as expressions => test_goldenscript }
+    test_each_path! { in "src/sql/testscripts/schema" as schema => test_goldenscript }
+    test_each_path! { in "src/sql/testscripts/expressions" as expressions => test_goldenscript_expr }
 
     fn test_goldenscript(path: &std::path::Path) {
+        // Since the runner's Session can't reference an Engine stored in the
+        // same struct, we pass in the session. Use both a BitCask and a Memory
+        // engine, and mirror operations across them. Emit engine operations to
+        // op_rx.
+        let (op_tx, op_rx) = crossbeam::channel::unbounded();
+        let tempdir = tempfile::TempDir::with_prefix("toydb").expect("tempdir failed");
+        let bitcask =
+            storage::BitCask::new(tempdir.path().join("bitcask")).expect("bitcask failed");
+        let memory = storage::Memory::new();
+        let engine = Local::new(Emit::new(Mirror::new(bitcask, memory), op_tx));
+        let mut runner = SQLRunner::new(&engine, op_rx);
+
+        goldenscript::run(&mut runner, path).expect("goldenscript failed")
+    }
+
+    fn test_goldenscript_expr(path: &std::path::Path) {
         goldenscript::run(&mut ExpressionRunner::new(), path).expect("goldenscript failed")
     }
 
+    /// A SQL test runner.
+    struct SQLRunner<'a> {
+        engine: &'a TestEngine,
+        session: Session<'a, TestEngine>,
+        op_rx: Receiver<Operation>,
+    }
+
+    type TestEngine = Local<Emit<Mirror<storage::BitCask, storage::Memory>>>;
+
+    impl<'a> SQLRunner<'a> {
+        fn new(engine: &'a TestEngine, op_rx: Receiver<Operation>) -> Self {
+            let session = engine.session();
+            Self { engine, session, op_rx }
+        }
+    }
+
+    impl<'a> goldenscript::Runner for SQLRunner<'a> {
+        fn run(&mut self, command: &goldenscript::Command) -> Result<String, Box<dyn Error>> {
+            let mut output = String::new();
+
+            // Handle runner commands.
+            match command.name.as_str() {
+                // dump
+                "dump" => {
+                    command.consume_args().reject_rest()?;
+                    let mut engine = self.engine.mvcc.engine.lock().expect("mutex failed");
+                    let mut iter = engine.scan(..);
+                    while let Some((key, value)) = iter.next().transpose()? {
+                        writeln!(
+                            output,
+                            "{} [{}]",
+                            format::MVCC::<format::SQL>::key_value(&key, &value),
+                            format::Raw::key_value(&key, &value)
+                        )?;
+                    }
+                    return Ok(output);
+                }
+
+                // schema [TABLE...]
+                "schema" => {
+                    let mut args = command.consume_args();
+                    let tables = args.rest_pos().iter().map(|arg| arg.value.clone()).collect_vec();
+                    args.reject_rest()?;
+
+                    let schemas = if tables.is_empty() {
+                        self.session.with_txn(true, |txn| txn.list_tables())?
+                    } else {
+                        tables
+                            .into_iter()
+                            .map(|t| self.session.with_txn(true, |txn| txn.must_get_table(&t)))
+                            .collect::<Result<_, _>>()?
+                    };
+                    return Ok(schemas.into_iter().map(|s| s.to_string()).join("\n"));
+                }
+
+                // Otherwise, fall through to SQL execution.
+                _ => {}
+            }
+
+            // The entire command is the statement to execute. There are no args.
+            if !command.args.is_empty() {
+                return Err("expressions should be given as a command with no args".into());
+            }
+            let input = &command.name;
+            let mut tags = command.tags.clone();
+
+            // Execute the statement.
+            let result = self.session.execute(input)?;
+
+            // Output the result if requested.
+            if tags.remove("result") {
+                writeln!(output, "{result:?}")?;
+            }
+
+            // Output engine ops if requested.
+            if tags.remove("ops") {
+                while let Ok(op) = self.op_rx.try_recv() {
+                    match op {
+                        Operation::Delete { key } => writeln!(
+                            output,
+                            "storage delete {} [{}]",
+                            format::MVCC::<format::SQL>::key(&key),
+                            format::Raw::key(&key),
+                        )?,
+                        Operation::Flush => writeln!(output, "storage flush")?,
+                        Operation::Set { key, value } => writeln!(
+                            output,
+                            "storage set {} [{}]",
+                            format::MVCC::<format::SQL>::key_value(&key, &value),
+                            format::Raw::key_value(&key, &value),
+                        )?,
+                    }
+                }
+            }
+
+            // Reject unknown tags.
+            if let Some(tag) = tags.iter().next() {
+                return Err(format!("unknown tag {tag}").into());
+            }
+
+            Ok(output)
+        }
+
+        /// If requested via [ops] tag, output engine operations for the command.
+        fn end_command(&mut self, _: &goldenscript::Command) -> Result<String, Box<dyn Error>> {
+            // Drain unconsumed operations.
+            while self.op_rx.try_recv().is_ok() {}
+            Ok(String::new())
+        }
+    }
+
+    /// A test runner for expressions specifically. Evaluates expressions to
+    /// values, and can optionally emit the expression tree.
     struct ExpressionRunner {
         engine: Local<storage::Memory>,
     }

src/sql/testscripts/schema/create_table

@@ -0,0 +1,43 @@
+# Tests basic CREATE TABLE functionality.
+
+# The result contains the table name. The table is written to storage.
+[result,ops]> CREATE TABLE test (id INTEGER PRIMARY KEY)
+---
+CreateTable { name: "test" }
+storage set mvcc:NextVersion → 2 ["\x00" → "\x02"]
+storage set mvcc:TxnActive(1) → "" ["\x01\x00\x00\x00\x00\x00\x00\x00\x01" → ""]
+storage set mvcc:TxnWrite(1, sql:Table("test")) → "" ["\x03\x00\x00\x00\x00\x00\x00\x00\x01\x00\xfftest\x00\xff\x00\xff\x00\x00" → ""]
+storage set mvcc:Version(sql:Table("test"), 1) → CREATE TABLE test ( id INTEGER PRIMARY KEY ) ["\x04\x00\xfftest\x00\xff\x00\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01" → "\x01\x10\x04test\x00\x01\x02id\x01\x00\x00\x01\x00\x00"]
+storage delete mvcc:TxnWrite(1, sql:Table("test")) ["\x03\x00\x00\x00\x00\x00\x00\x00\x01\x00\xfftest\x00\xff\x00\xff\x00\x00"]
+storage delete mvcc:TxnActive(1) ["\x01\x00\x00\x00\x00\x00\x00\x00\x01"]
+
+dump
+---
+mvcc:NextVersion → 2 ["\x00" → "\x02"]
+mvcc:Version(sql:Table("test"), 1) → CREATE TABLE test ( id INTEGER PRIMARY KEY ) ["\x04\x00\xfftest\x00\xff\x00\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01" → "\x01\x10\x04test\x00\x01\x02id\x01\x00\x00\x01\x00\x00"]
+
+# Errors if table already exists.
+!> CREATE TABLE test (id INTEGER PRIMARY KEY)
+---
+Error: invalid input: table test already exists
+
+# No table name or columns errors.
+!> CREATE TABLE
+!> CREATE TABLE name
+!> CREATE TABLE name ()
+---
+Error: invalid input: unexpected end of input
+Error: invalid input: unexpected end of input
+Error: invalid input: expected identifier, got )
+
+# Missing table or column names error.
+!> CREATE TABLE (id INTEGER PRIMARY KEY)
+!> CREATE TABLE name (INTEGER PRIMARY KEY)
+---
+Error: invalid input: expected identifier, got (
+Error: invalid input: expected identifier, got INTEGER
+
+# Unterminated identifier errors.
+!> CREATE TABLE "name (id INTEGER PRIMARY KEY)
+---
+Error: invalid input: unexpected end of quoted identifier

src/sql/testscripts/schema/create_table_datatypes

@@ -0,0 +1,38 @@
+# Tests CREATE TABLE datatypes.
+
+# Create columns with all datatypes.
+> CREATE TABLE datatypes ( \
+    id INTEGER PRIMARY KEY, \
+    "bool" BOOL, \
+    "boolean" BOOLEAN, \
+    "double" DOUBLE, \
+    "float" FLOAT, \
+    "int" INT, \
+    "integer" INTEGER, \
+    "string" STRING, \
+    "text" TEXT, \
+    "varchar" VARCHAR \
+)
+schema
+---
+CREATE TABLE datatypes (
+  id INTEGER PRIMARY KEY,
+  "bool" BOOLEAN DEFAULT NULL,
+  "boolean" BOOLEAN DEFAULT NULL,
+  "double" FLOAT DEFAULT NULL,
+  "float" FLOAT DEFAULT NULL,
+  "int" INTEGER DEFAULT NULL,
+  "integer" INTEGER DEFAULT NULL,
+  "string" STRING DEFAULT NULL,
+  "text" STRING DEFAULT NULL,
+  "varchar" STRING DEFAULT NULL
+)
+
+# Missing or unknown datatype errors.
+!> CREATE TABLE test (id INTEGER PRIMARY KEY, value)
+!> CREATE TABLE test (id INTEGER PRIMARY KEY, value FOO)
+!> CREATE TABLE test (id INTEGER PRIMARY KEY, value INDEX)
+---
+Error: invalid input: unexpected token )
+Error: invalid input: unexpected token foo
+Error: invalid input: unexpected token INDEX

src/sql/testscripts/schema/create_table_default

@@ -0,0 +1,54 @@
+# Tests column defaults.
+
+# All datatypes.
+> CREATE TABLE datatypes ( \
+    id INT PRIMARY KEY, \
+    "bool" BOOLEAN DEFAULT true, \
+    "float" FLOAT DEFAULT 3.14, \
+    "int" INTEGER DEFAULT 7, \
+    "string" STRING DEFAULT 'foo' \
+)
+schema datatypes
+---
+CREATE TABLE datatypes (
+  id INTEGER PRIMARY KEY,
+  "bool" BOOLEAN DEFAULT TRUE,
+  "float" FLOAT DEFAULT 3.14,
+  "int" INTEGER DEFAULT 7,
+  "string" STRING DEFAULT foo
+)
+
+# Default datatypes must match column. This includes float/integer types.
+!> CREATE TABLE name (id INT PRIMARY KEY, value STRING DEFAULT 7)
+!> CREATE TABLE name (id INT PRIMARY KEY, value INTEGER DEFAULT 3.14)
+!> CREATE TABLE name (id INT PRIMARY KEY, value FLOAT DEFAULT 7)
+---
+Error: invalid input: invalid datatype INTEGER for STRING column value
+Error: invalid input: invalid datatype FLOAT for INTEGER column value
+Error: invalid input: invalid datatype INTEGER for FLOAT column value
+
+# Default values can be expressions.
+> CREATE TABLE expr (id INT PRIMARY KEY, value INT DEFAULT 7 + 3 * 2)
+schema expr
+---
+CREATE TABLE expr (
+  id INTEGER PRIMARY KEY,
+  value INTEGER DEFAULT 13
+)
+
+# NULL is a value default for a nullable column (and is the implicit default).
+> CREATE TABLE "nullable" (id INT PRIMARY KEY, value STRING DEFAULT NULL, implicit STRING)
+schema nullable
+---
+CREATE TABLE nullable (
+  id INTEGER PRIMARY KEY,
+  value STRING DEFAULT NULL,
+  implicit STRING DEFAULT NULL
+)
+
+# A NULL default errors for a non-nullable column, including primary keys.
+!> CREATE TABLE name (id INT PRIMARY KEY DEFAULT NULL)
+!> CREATE TABLE name (id INT PRIMARY KEY, value STRING NOT NULL DEFAULT NULL)
+---
+Error: invalid input: invalid NULL default for non-nullable column id
+Error: invalid input: invalid NULL default for non-nullable column value