feat(katana-db): db migration from older version

crates/katana/storage/db/Cargo.toml

@@ -14,8 +14,9 @@ page_size = "0.6.0"
 parking_lot.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-tempfile = { version = "3.8.1", optional = true }
+tempfile = "3.8.1"
 thiserror.workspace = true
+tracing.workspace = true
 
 cairo-vm.workspace = true
 roaring = { version = "0.10.3", features = [ "serde" ] }
@@ -37,12 +38,11 @@ rev = "b34b0d3"
 cairo-lang-starknet.workspace = true
 criterion = "0.5.1"
 starknet.workspace = true
-tempfile = "3.8.1"
 
 [features]
 default = [ "postcard" ]
 postcard = [ "dep:postcard" ]
-test-utils = [ "dep:tempfile" ]
+test-utils = [  ]
 
 [[bench]]
 harness = false

crates/katana/storage/db/src/error.rs

@@ -6,8 +6,8 @@ pub enum DatabaseError {
     #[error(transparent)]
     Codec(#[from] CodecError),
 
-    #[error("failed to create db table: {0}")]
-    CreateTable(libmdbx::Error),
+    #[error("failed to create db table '{table}': {error}")]
+    CreateTable { table: &'static str, error: libmdbx::Error },
 
     #[error("failed to commit db transaction: {0}")]
     Commit(libmdbx::Error),

crates/katana/storage/db/src/lib.rs

@@ -8,17 +8,21 @@ use anyhow::{anyhow, Context};
 pub mod codecs;
 pub mod error;
 pub mod mdbx;
+pub mod migration;
 pub mod models;
 pub mod tables;
 pub mod utils;
 pub mod version;
 
 use mdbx::{DbEnv, DbEnvKind};
 use tables::Schema;
+use tracing::{info, warn};
 use utils::is_database_empty;
 pub use version::CURRENT_DB_VERSION;
 use version::{check_db_version, create_db_version_file, DatabaseVersionError};
 
+pub(crate) const LOG_TARGET: &str = "katana::db";
+
 /// Initialize the database at the given path and returning a handle to the its
 /// environment.
 ///
@@ -41,8 +45,14 @@ pub(crate) fn init_db_with_schema<S: Schema>(path: impl AsRef<Path>) -> anyhow::
             format!("Inserting database version file at path {}", path.as_ref().display())
         })?
     } else {
-        match check_db_version(&path) {
+        match check_db_version::<S>(&path) {
             Ok(_) => {}
+            Err(DatabaseVersionError::MismatchVersion { expected, found }) => {
+                warn!(target: LOG_TARGET, %expected, %found, "Mismatch database version.");
+
+                info!(target: LOG_TARGET, "Attempting to migrate database.");
+                migration::migrate_db(path.as_ref()).context("Migrating database")?;
+            }
             Err(DatabaseVersionError::FileNotFound) => create_db_version_file(&path, S::VERSION)
                 .with_context(|| {
                     format!(

crates/katana/storage/db/src/mdbx/mod.rs

@@ -79,20 +79,7 @@ impl<S: Schema> DbEnv<S> {
 
     /// Creates all the defined tables in [`Tables`], if necessary.
     pub fn create_tables(&self) -> Result<(), DatabaseError> {
-        let tx = self.inner.begin_rw_txn().map_err(DatabaseError::CreateRWTx)?;
-
-        for table in S::all() {
-            let flags = match table.table_type() {
-                TableType::Table => DatabaseFlags::default(),
-                TableType::DupSort => DatabaseFlags::DUP_SORT,
-            };
-
-            tx.create_db(Some(table.name()), flags).map_err(DatabaseError::CreateTable)?;
-        }
-
-        tx.commit().map_err(DatabaseError::Commit)?;
-
-        Ok(())
+        self.create_tables_from_schema::<S>()
     }
 
     /// Begin a read-only transaction.
@@ -128,6 +115,26 @@ impl<S: Schema> DbEnv<S> {
         tx.commit()?;
         Ok(res)
     }
+
+    /// Creates all the defined tables in [`Tables`], if necessary.
+    pub(crate) fn create_tables_from_schema<R: Schema>(&self) -> Result<(), DatabaseError> {
+        let tx = self.inner.begin_rw_txn().map_err(DatabaseError::CreateRWTx)?;
+
+        for table in R::all() {
+            let flags = match table.table_type() {
+                TableType::Table => DatabaseFlags::default(),
+                TableType::DupSort => DatabaseFlags::DUP_SORT,
+            };
+
+            let name = table.name();
+            tx.create_db(Some(name), flags)
+                .map_err(|error| DatabaseError::CreateTable { table: name, error })?;
+        }
+
+        tx.commit().map_err(DatabaseError::Commit)?;
+
+        Ok(())
+    }
 }
 
 #[cfg(any(test, feature = "test-utils"))]
@@ -165,8 +172,10 @@ mod tests {
     use crate::codecs::Encode;
     use crate::mdbx::cursor::Walker;
     use crate::mdbx::test_utils::create_test_db;
-    use crate::models::storage::StorageEntry;
-    use crate::tables::{BlockHashes, ContractInfo, ContractStorage, Headers, Table};
+    use crate::models::contract::ContractNonceChange;
+    use crate::models::list::BlockList;
+    use crate::models::storage::{ContractStorageKey, StorageEntry};
+    use crate::tables::{self, BlockHashes, ContractInfo, ContractStorage, Headers, Table};
 
     const ERROR_PUT: &str = "Not able to insert value into table.";
     const ERROR_DELETE: &str = "Failed to delete value from table.";
@@ -422,4 +431,33 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn db_get_put_unchecked() {
+        // create a database with the v1 schema
+        let env = create_test_db(DbEnvKind::RW);
+        let tx = env.tx_mut().expect(ERROR_INIT_TX);
+
+        // drop a table first bcs database already reached max tables limit
+        unsafe { tx.drop_table::<tables::NonceChangeHistory>().unwrap() }
+
+        // get a value from an existing table from the database will not return error
+        let result = tx.get_unchecked::<tables::StorageChangeSet>(ContractStorageKey::default());
+        assert!(result.is_ok());
+
+        // get a value from a nonexistent table from the database will return error
+        let result = tx.get_unchecked::<tables::v0::NonceChanges>(1);
+        assert_eq!(result, Err(DatabaseError::OpenDb(libmdbx::Error::NotFound)));
+
+        // put a value into an existing table in the database will not return error
+        let key = ContractStorageKey::default();
+        let val = BlockList::new();
+        let result = tx.put_unchecked::<tables::StorageChangeSet>(key, val);
+        assert!(result.is_ok());
+
+        // put a value into a nonexistent table in the database will return error
+        let val = ContractNonceChange::new(felt!("0x1").into(), felt!("0x1"));
+        let result = tx.put_unchecked::<tables::v0::NonceChanges>(1, val);
+        assert_eq!(result, Err(DatabaseError::OpenDb(libmdbx::Error::NotFound)));
+    }
 }

crates/katana/storage/db/src/mdbx/tx.rs

@@ -23,7 +23,7 @@ pub type TxRW = Tx<libmdbx::RW, SchemaV1>;
 #[derive(Debug)]
 pub struct Tx<K: TransactionKind, S: Schema> {
     /// Libmdbx-sys transaction.
-    inner: libmdbx::Transaction<K>,
+    pub(crate) inner: libmdbx::Transaction<K>,
     /// Marker for the db schema.
     _schema: std::marker::PhantomData<S>,
     // the array size is hardcoded to the number of tables in current db version for now. ideally
@@ -51,6 +51,12 @@ where
             .map_err(DatabaseError::CreateCursor)
     }
 
+    /// Creates a cursor to iterate over a table items.
+    pub fn cursor_unchecked<T: Table>(&self) -> Result<Cursor<K, T>, DatabaseError> {
+        let dbi = self.inner.open_db(Some(T::NAME)).map_err(DatabaseError::OpenDb)?.dbi();
+        self.inner.cursor_with_dbi(dbi).map(Cursor::new).map_err(DatabaseError::CreateCursor)
+    }
+
     /// Gets a table database handle if it exists, otherwise creates it.
     pub fn get_dbi<T: Table>(&self) -> Result<DBI, DatabaseError> {
         // SAFETY:
@@ -124,6 +130,18 @@ impl<S: Schema> Tx<RW, S> {
         Ok(())
     }
 
+    pub fn put_unchecked<T: Table>(
+        &self,
+        key: T::Key,
+        value: T::Value,
+    ) -> Result<(), DatabaseError> {
+        let key = key.encode();
+        let value = value.compress();
+        let dbi = self.inner.open_db(Some(T::NAME)).map_err(DatabaseError::OpenDb)?.dbi();
+        self.inner.put(dbi, key, value, WriteFlags::UPSERT).unwrap();
+        Ok(())
+    }
+
     /// Delete items from a database, removing the key/data pair if it exists.
     ///
     /// If the data parameter is [Some] only the matching data item will be deleted. Otherwise, if