feat(torii): model upgrades (#2637)

* feat(torii): model upgrades

* upgrade processor for event and model

* fix processors

* fix: model upgrade

* wrap up model upgrade

* ftm

* clippy

* fmt

* refactor: diff

* fmtg

* fix: set model cache

* fix add model members

* fix

* feat: shared cache between grpc & engine and fix partial deser

* fix: test and fmt

* fix

* primitives not option

* fix: enums

* Revert "fix: enums"

This reverts commit d86fcc4.

* Revert "primitives not option"

This reverts commit 210ac31.

* fix enum sql value

* main

* remove prints & format

* fix quer ytest

* fix: bool

* fix: ararys

* fmt

* fix: map row to ty

* fix: enum

* fix: enum

* fmt

* fix: primitive len

* Revert "fix: primitive len"

This reverts commit 551c6bd.

* refactotr: dont use modelr eader block

* Revert "Revert "fix: primitive len""

This reverts commit cea8102.

bin/torii/src/main.rs

@@ -35,6 +35,7 @@ use torii_core::executor::Executor;
 use torii_core::processors::store_transaction::StoreTransactionProcessor;
 use torii_core::processors::EventProcessorConfig;
 use torii_core::simple_broker::SimpleBroker;
+use torii_core::sql::cache::ModelCache;
 use torii_core::sql::Sql;
 use torii_core::types::{Contract, ContractType, Model};
 use torii_server::proxy::Proxy;
@@ -113,7 +114,9 @@ async fn main() -> anyhow::Result<()> {
         executor.run().await.unwrap();
     });
 
-    let db = Sql::new(pool.clone(), sender.clone(), &args.indexing.contracts).await?;
+    let model_cache = Arc::new(ModelCache::new(pool.clone()));
+    let db = Sql::new(pool.clone(), sender.clone(), &args.indexing.contracts, model_cache.clone())
+        .await?;
 
     let processors = Processors {
         transaction: vec![Box::new(StoreTransactionProcessor)],
@@ -153,9 +156,15 @@ async fn main() -> anyhow::Result<()> {
     );
 
     let shutdown_rx = shutdown_tx.subscribe();
-    let (grpc_addr, grpc_server) =
-        torii_grpc::server::new(shutdown_rx, &pool, block_rx, world_address, Arc::clone(&provider))
-            .await?;
+    let (grpc_addr, grpc_server) = torii_grpc::server::new(
+        shutdown_rx,
+        &pool,
+        block_rx,
+        world_address,
+        Arc::clone(&provider),
+        model_cache,
+    )
+    .await?;
 
     let mut libp2p_relay_server = torii_relay::server::Relay::new(
         db,

crates/dojo/types/src/primitive.rs

@@ -190,46 +190,29 @@ impl Primitive {
         }
     }
 
-    pub fn to_sql_value(&self) -> Result<String, PrimitiveError> {
-        let value = self.serialize()?;
-
-        if value.is_empty() {
-            return Err(PrimitiveError::MissingFieldElement);
-        }
-
+    pub fn to_sql_value(&self) -> String {
         match self {
             // Integers
-            Primitive::I8(_) => Ok(format!("{}", try_from_felt::<i8>(value[0])?)),
-            Primitive::I16(_) => Ok(format!("{}", try_from_felt::<i16>(value[0])?)),
-            Primitive::I32(_) => Ok(format!("{}", try_from_felt::<i32>(value[0])?)),
-            Primitive::I64(_) => Ok(format!("{}", try_from_felt::<i64>(value[0])?)),
+            Primitive::I8(i8) => format!("{}", i8.unwrap_or_default()),
+            Primitive::I16(i16) => format!("{}", i16.unwrap_or_default()),
+            Primitive::I32(i32) => format!("{}", i32.unwrap_or_default()),
+            Primitive::I64(i64) => format!("{}", i64.unwrap_or_default()),
 
-            Primitive::U8(_)
-            | Primitive::U16(_)
-            | Primitive::U32(_)
-            | Primitive::USize(_)
-            | Primitive::Bool(_) => Ok(format!("{}", value[0])),
+            Primitive::U8(u8) => format!("{}", u8.unwrap_or_default()),
+            Primitive::U16(u16) => format!("{}", u16.unwrap_or_default()),
+            Primitive::U32(u32) => format!("{}", u32.unwrap_or_default()),
+            Primitive::USize(u32) => format!("{}", u32.unwrap_or_default()),
+            Primitive::Bool(bool) => format!("{}", bool.unwrap_or_default() as i32),
 
             // Hex string
-            Primitive::I128(_) => Ok(format!("{:#064x}", try_from_felt::<i128>(value[0])?)),
-            Primitive::ContractAddress(_)
-            | Primitive::ClassHash(_)
-            | Primitive::Felt252(_)
-            | Primitive::U128(_)
-            | Primitive::U64(_) => Ok(format!("{:#064x}", value[0])),
-
-            Primitive::U256(_) => {
-                if value.len() < 2 {
-                    Err(PrimitiveError::NotEnoughFieldElements)
-                } else {
-                    let mut buffer = [0u8; 32];
-                    let value0_bytes = value[0].to_bytes_be();
-                    let value1_bytes = value[1].to_bytes_be();
-                    buffer[16..].copy_from_slice(&value0_bytes[16..]);
-                    buffer[..16].copy_from_slice(&value1_bytes[16..]);
-                    Ok(format!("0x{}", hex::encode(buffer)))
-                }
-            }
+            Primitive::I128(i128) => format!("0x{:064x}", i128.unwrap_or_default()),
+            Primitive::ContractAddress(felt) => format!("0x{:064x}", felt.unwrap_or_default()),
+            Primitive::ClassHash(felt) => format!("0x{:064x}", felt.unwrap_or_default()),
+            Primitive::Felt252(felt) => format!("0x{:064x}", felt.unwrap_or_default()),
+            Primitive::U128(u128) => format!("0x{:064x}", u128.unwrap_or_default()),
+            Primitive::U64(u64) => format!("0x{:064x}", u64.unwrap_or_default()),
+
+            Primitive::U256(u256) => format!("0x{:064x}", u256.unwrap_or_default()),
         }
     }
 
@@ -436,7 +419,7 @@ mod tests {
         let primitive = Primitive::U256(Some(U256::from_be_hex(
             "aaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbccccccccccccccccdddddddddddddddd",
         )));
-        let sql_value = primitive.to_sql_value().unwrap();
+        let sql_value = primitive.to_sql_value();
         let serialized = primitive.serialize().unwrap();
 
         let mut deserialized = primitive;

crates/dojo/types/src/schema.rs

@@ -173,6 +173,11 @@ impl Ty {
     }
 
     pub fn deserialize(&mut self, felts: &mut Vec<Felt>) -> Result<(), PrimitiveError> {
+        if felts.is_empty() {
+            // return early if there are no felts to deserialize
+            return Ok(());
+        }
+
         match self {
             Ty::Primitive(c) => {
                 c.deserialize(felts)?;
@@ -224,6 +229,85 @@ impl Ty {
         }
         Ok(())
     }
+
+    /// Returns a new Ty containing only the differences between self and other
+    pub fn diff(&self, other: &Ty) -> Option<Ty> {
+        match (self, other) {
+            (Ty::Struct(s1), Ty::Struct(s2)) => {
+                // Find members that exist in s1 but not in s2, or are different
+                let diff_children: Vec<Member> = s1
+                    .children
+                    .iter()
+                    .filter(|m1| {
+                        s2.children
+                            .iter()
+                            .find(|m2| m2.name == m1.name)
+                            .map_or(true, |m2| *m1 != m2)
+                    })
+                    .cloned()
+                    .collect();
+
+                if diff_children.is_empty() {
+                    None
+                } else {
+                    Some(Ty::Struct(Struct { name: s1.name.clone(), children: diff_children }))
+                }
+            }
+            (Ty::Enum(e1), Ty::Enum(e2)) => {
+                // Find options that exist in e1 but not in e2, or are different
+                let diff_options: Vec<EnumOption> = e1
+                    .options
+                    .iter()
+                    .filter(|o1| {
+                        e2.options.iter().find(|o2| o2.name == o1.name).map_or(true, |o2| *o1 != o2)
+                    })
+                    .cloned()
+                    .collect();
+
+                if diff_options.is_empty() {
+                    None
+                } else {
+                    Some(Ty::Enum(Enum {
+                        name: e1.name.clone(),
+                        option: e1.option,
+                        options: diff_options,
+                    }))
+                }
+            }
+            (Ty::Array(a1), Ty::Array(a2)) => {
+                if a1 == a2 {
+                    None
+                } else {
+                    Some(Ty::Array(a1.clone()))
+                }
+            }
+            (Ty::Tuple(t1), Ty::Tuple(t2)) => {
+                if t1 == t2 {
+                    None
+                } else {
+                    Some(Ty::Tuple(t1.clone()))
+                }
+            }
+            (Ty::ByteArray(b1), Ty::ByteArray(b2)) => {
+                if b1 == b2 {
+                    None
+                } else {
+                    Some(Ty::ByteArray(b1.clone()))
+                }
+            }
+            (Ty::Primitive(p1), Ty::Primitive(p2)) => {
+                if p1 == p2 {
+                    None
+                } else {
+                    Some(Ty::Primitive(*p1))
+                }
+            }
+            // Different types entirely - we cannot diff them
+            _ => {
+                panic!("Type mismatch between self {:?} and other {:?}", self.name(), other.name())
+            }
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -351,8 +435,8 @@ impl Enum {
         }
     }
 
-    pub fn to_sql_value(&self) -> Result<String, EnumError> {
-        self.option().map(|option| option.name.clone())
+    pub fn to_sql_value(&self) -> String {
+        self.option().unwrap_or(&self.options[0]).name.clone()
     }
 }
 
@@ -597,4 +681,77 @@ mod tests {
             assert_eq!(format_member(&member), expected);
         }
     }
+
+    #[test]
+    fn test_ty_diff() {
+        // Test struct diff
+        let struct1 = Ty::Struct(Struct {
+            name: "TestStruct".to_string(),
+            children: vec![
+                Member {
+                    name: "field1".to_string(),
+                    ty: Ty::Primitive(Primitive::U32(None)),
+                    key: false,
+                },
+                Member {
+                    name: "field2".to_string(),
+                    ty: Ty::Primitive(Primitive::U32(None)),
+                    key: false,
+                },
+                Member {
+                    name: "field3".to_string(),
+                    ty: Ty::Primitive(Primitive::U32(None)),
+                    key: false,
+                },
+            ],
+        });
+
+        let struct2 = Ty::Struct(Struct {
+            name: "TestStruct".to_string(),
+            children: vec![Member {
+                name: "field1".to_string(),
+                ty: Ty::Primitive(Primitive::U32(None)),
+                key: false,
+            }],
+        });
+
+        // Should show only field2 and field3 as differences
+        let diff = struct1.diff(&struct2).unwrap();
+        if let Ty::Struct(s) = diff {
+            assert_eq!(s.children.len(), 2);
+            assert_eq!(s.children[0].name, "field2");
+            assert_eq!(s.children[1].name, "field3");
+        } else {
+            panic!("Expected Struct diff");
+        }
+
+        // Test enum diff
+        let enum1 = Ty::Enum(Enum {
+            name: "TestEnum".to_string(),
+            option: None,
+            options: vec![
+                EnumOption { name: "Option1".to_string(), ty: Ty::Tuple(vec![]) },
+                EnumOption { name: "Option2".to_string(), ty: Ty::Tuple(vec![]) },
+            ],
+        });
+
+        let enum2 = Ty::Enum(Enum {
+            name: "TestEnum".to_string(),
+            option: None,
+            options: vec![EnumOption { name: "Option1".to_string(), ty: Ty::Tuple(vec![]) }],
+        });
+
+        // Should show only Option2 as difference
+        let diff = enum1.diff(&enum2).unwrap();
+        if let Ty::Enum(e) = diff {
+            assert_eq!(e.options.len(), 1);
+            assert_eq!(e.options[0].name, "Option2");
+        } else {
+            panic!("Expected Enum diff");
+        }
+
+        // Test no differences
+        let same_struct = struct2.diff(&struct2);
+        assert!(same_struct.is_none());
+    }
 }

crates/dojo/world/src/contracts/model.rs

@@ -5,7 +5,7 @@ use cainome::cairo_serde::{CairoSerde as _, ContractAddress, Error as CainomeErr
 use dojo_types::packing::{PackingError, ParseError};
 use dojo_types::primitive::{Primitive, PrimitiveError};
 use dojo_types::schema::{Enum, EnumOption, Member, Struct, Ty};
-use starknet::core::types::Felt;
+use starknet::core::types::{BlockId, Felt};
 use starknet::core::utils::{
     cairo_short_string_to_felt, parse_cairo_short_string, CairoShortStringToFeltError,
     NonAsciiNameError, ParseCairoShortStringError,
@@ -86,13 +86,22 @@ where
         namespace: &str,
         name: &str,
         world: &'a WorldContractReader<P>,
+    ) -> Result<ModelRPCReader<'a, P>, ModelError> {
+        Self::new_with_block(namespace, name, world, world.block_id).await
+    }
+
+    pub async fn new_with_block(
+        namespace: &str,
+        name: &str,
+        world: &'a WorldContractReader<P>,
+        block_id: BlockId,
     ) -> Result<ModelRPCReader<'a, P>, ModelError> {
         let model_selector = naming::compute_selector_from_names(namespace, name);
 
         // Events are also considered like models from a off-chain perspective. They both have
         // introspection and convey type information.
         let (contract_address, class_hash) =
-            match world.resource(&model_selector).block_id(world.block_id).call().await? {
+            match world.resource(&model_selector).block_id(block_id).call().await? {
                 abigen::world::Resource::Model((address, hash)) => (address, hash),
                 abigen::world::Resource::Event((address, hash)) => (address, hash),
                 _ => return Err(ModelError::ModelNotFound),
@@ -104,7 +113,8 @@ where
             return Err(ModelError::ModelNotFound);
         }
 
-        let model_reader = ModelContractReader::new(contract_address.into(), world.provider());
+        let mut model_reader = ModelContractReader::new(contract_address.into(), world.provider());
+        model_reader.set_block(block_id);
 
         Ok(Self {
             namespace: namespace.into(),
@@ -176,7 +186,7 @@ where
         parse_schema(&abigen::model::Ty::Struct(res)).map_err(ModelError::Parse)
     }
 
-    // For non fixed layouts, packed and unpacked sizes are None.
+    // For non fixed layouts,   packed and unpacked sizes are None.
     // Therefore we return 0 in this case.
     async fn packed_size(&self) -> Result<u32, ModelError> {
         Ok(self.model_reader.packed_size().call().await?.unwrap_or(0))

crates/dojo/world/src/contracts/world.rs

@@ -1,5 +1,6 @@
 use std::result::Result;
 
+use starknet::core::types::BlockId;
 use starknet::providers::Provider;
 
 pub use super::abigen::world::{
@@ -33,4 +34,13 @@ where
     ) -> Result<ModelRPCReader<'_, P>, ModelError> {
         ModelRPCReader::new(namespace, name, self).await
     }
+
+    pub async fn model_reader_with_block(
+        &self,
+        namespace: &str,
+        name: &str,
+        block_id: BlockId,
+    ) -> Result<ModelRPCReader<'_, P>, ModelError> {
+        ModelRPCReader::new_with_block(namespace, name, self, block_id).await
+    }
 }

crates/sozo/ops/src/model.rs

@@ -412,7 +412,7 @@ fn format_primitive(
     let mut _p = *p;
     let _ = _p.deserialize(values);
 
-    format!("{}{}", _start_indent(level, start_indent), _p.to_sql_value().unwrap())
+    format!("{}{}", _start_indent(level, start_indent), _p.to_sql_value())
 }
 
 fn format_byte_array(values: &mut Vec<Felt>, level: usize, start_indent: bool) -> String {

crates/torii/core/src/engine.rs

@@ -37,6 +37,8 @@ use crate::processors::store_del_record::StoreDelRecordProcessor;
 use crate::processors::store_set_record::StoreSetRecordProcessor;
 use crate::processors::store_update_member::StoreUpdateMemberProcessor;
 use crate::processors::store_update_record::StoreUpdateRecordProcessor;
+use crate::processors::upgrade_event::UpgradeEventProcessor;
+use crate::processors::upgrade_model::UpgradeModelProcessor;
 use crate::processors::{
     BlockProcessor, EventProcessor, EventProcessorConfig, TransactionProcessor,
 };
@@ -76,6 +78,8 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Processors<P> {
                 vec![
                     Box::new(RegisterModelProcessor) as Box<dyn EventProcessor<P>>,
                     Box::new(RegisterEventProcessor) as Box<dyn EventProcessor<P>>,
+                    Box::new(UpgradeModelProcessor) as Box<dyn EventProcessor<P>>,
+                    Box::new(UpgradeEventProcessor) as Box<dyn EventProcessor<P>>,
                     Box::new(StoreSetRecordProcessor),
                     Box::new(StoreDelRecordProcessor),
                     Box::new(StoreUpdateRecordProcessor),