feat(torii-core): json value for ty for efficient json ser (#2730)

crates/dojo/types/Cargo.toml

@@ -20,3 +20,4 @@ starknet-crypto.workspace = true
 strum.workspace = true
 strum_macros.workspace = true
 thiserror.workspace = true
+indexmap.workspace = true

crates/dojo/types/src/schema.rs

@@ -1,9 +1,13 @@
 use std::any::type_name;
+use std::str::FromStr;
 
 use cainome::cairo_serde::{ByteArray, CairoSerde};
+use crypto_bigint::{Encoding, U256};
+use indexmap::IndexMap;
 use itertools::Itertools;
 use num_traits::ToPrimitive;
 use serde::{Deserialize, Serialize};
+use serde_json::{json, Value as JsonValue};
 use starknet::core::types::Felt;
 use strum_macros::AsRefStr;
 
@@ -308,6 +312,196 @@ impl Ty {
             }
         }
     }
+
+    /// Convert a Ty to a JSON Value
+    pub fn to_json_value(&self) -> Result<JsonValue, PrimitiveError> {
+        match self {
+            Ty::Primitive(primitive) => match primitive {
+                Primitive::Bool(Some(v)) => Ok(json!(*v)),
+                Primitive::I8(Some(v)) => Ok(json!(*v)),
+                Primitive::I16(Some(v)) => Ok(json!(*v)),
+                Primitive::I32(Some(v)) => Ok(json!(*v)),
+                Primitive::I64(Some(v)) => Ok(json!(v.to_string())),
+                Primitive::I128(Some(v)) => Ok(json!(v.to_string())),
+                Primitive::U8(Some(v)) => Ok(json!(*v)),
+                Primitive::U16(Some(v)) => Ok(json!(*v)),
+                Primitive::U32(Some(v)) => Ok(json!(*v)),
+                Primitive::U64(Some(v)) => Ok(json!(v.to_string())),
+                Primitive::U128(Some(v)) => Ok(json!(v.to_string())),
+                Primitive::USize(Some(v)) => Ok(json!(*v)),
+                Primitive::U256(Some(v)) => {
+                    let bytes = v.to_be_bytes();
+                    let high = u128::from_be_bytes(bytes[..16].try_into().unwrap());
+                    let low = u128::from_be_bytes(bytes[16..].try_into().unwrap());
+                    Ok(json!({
+                        "high": high.to_string(),
+                        "low": low.to_string()
+                    }))
+                }
+                Primitive::Felt252(Some(v)) => Ok(json!(format!("{:#x}", v))),
+                Primitive::ClassHash(Some(v)) => Ok(json!(format!("{:#x}", v))),
+                Primitive::ContractAddress(Some(v)) => Ok(json!(format!("{:#x}", v))),
+                _ => Err(PrimitiveError::MissingFieldElement),
+            },
+            Ty::Struct(s) => {
+                let mut obj = IndexMap::new();
+                for member in &s.children {
+                    obj.insert(member.name.clone(), member.ty.to_json_value()?);
+                }
+                Ok(json!(obj))
+            }
+            Ty::Enum(e) => {
+                let option = e.option().map_err(|_| PrimitiveError::MissingFieldElement)?;
+                Ok(json!({
+                    option.name.clone(): option.ty.to_json_value()?
+                }))
+            }
+            Ty::Array(items) => {
+                let values: Result<Vec<_>, _> = items.iter().map(|ty| ty.to_json_value()).collect();
+                Ok(json!(values?))
+            }
+            Ty::Tuple(items) => {
+                let values: Result<Vec<_>, _> = items.iter().map(|ty| ty.to_json_value()).collect();
+                Ok(json!(values?))
+            }
+            Ty::ByteArray(bytes) => Ok(json!(bytes.clone())),
+        }
+    }
+
+    /// Parse a JSON Value into a Ty
+    pub fn from_json_value(&mut self, value: JsonValue) -> Result<(), PrimitiveError> {
+        match (self, value) {
+            (Ty::Primitive(primitive), value) => match primitive {
+                Primitive::Bool(v) => {
+                    if let JsonValue::Bool(b) = value {
+                        *v = Some(b);
+                    }
+                }
+                Primitive::I8(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_i64().map(|n| n as i8);
+                    }
+                }
+                Primitive::I16(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_i64().map(|n| n as i16);
+                    }
+                }
+                Primitive::I32(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_i64().map(|n| n as i32);
+                    }
+                }
+                Primitive::I64(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = s.parse().ok();
+                    }
+                }
+                Primitive::I128(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = s.parse().ok();
+                    }
+                }
+                Primitive::U8(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_u64().map(|n| n as u8);
+                    }
+                }
+                Primitive::U16(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_u64().map(|n| n as u16);
+                    }
+                }
+                Primitive::U32(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_u64().map(|n| n as u32);
+                    }
+                }
+                Primitive::U64(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = s.parse().ok();
+                    }
+                }
+                Primitive::U128(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = s.parse().ok();
+                    }
+                }
+                Primitive::USize(v) => {
+                    if let JsonValue::Number(n) = value {
+                        *v = n.as_u64().map(|n| n as u32);
+                    }
+                }
+                Primitive::U256(v) => {
+                    if let JsonValue::Object(obj) = value {
+                        if let (Some(JsonValue::String(high)), Some(JsonValue::String(low))) =
+                            (obj.get("high"), obj.get("low"))
+                        {
+                            if let (Ok(high), Ok(low)) = (high.parse::<u128>(), low.parse::<u128>())
+                            {
+                                let mut bytes = [0u8; 32];
+                                bytes[..16].copy_from_slice(&high.to_be_bytes());
+                                bytes[16..].copy_from_slice(&low.to_be_bytes());
+                                *v = Some(U256::from_be_slice(&bytes));
+                            }
+                        }
+                    }
+                }
+                Primitive::Felt252(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = Felt::from_str(&s).ok();
+                    }
+                }
+                Primitive::ClassHash(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = Felt::from_str(&s).ok();
+                    }
+                }
+                Primitive::ContractAddress(v) => {
+                    if let JsonValue::String(s) = value {
+                        *v = Felt::from_str(&s).ok();
+                    }
+                }
+            },
+            (Ty::Struct(s), JsonValue::Object(obj)) => {
+                for member in &mut s.children {
+                    if let Some(value) = obj.get(&member.name) {
+                        member.ty.from_json_value(value.clone())?;
+                    }
+                }
+            }
+            (Ty::Enum(e), JsonValue::Object(obj)) => {
+                if let Some((name, value)) = obj.into_iter().next() {
+                    e.set_option(&name).map_err(|_| PrimitiveError::TypeMismatch)?;
+                    if let Some(option) = e.option {
+                        e.options[option as usize].ty.from_json_value(value)?;
+                    }
+                }
+            }
+            (Ty::Array(items), JsonValue::Array(values)) => {
+                let template = items[0].clone();
+                items.clear();
+                for value in values {
+                    let mut item = template.clone();
+                    item.from_json_value(value)?;
+                    items.push(item);
+                }
+            }
+            (Ty::Tuple(items), JsonValue::Array(values)) => {
+                if items.len() != values.len() {
+                    return Err(PrimitiveError::TypeMismatch);
+                }
+                for (item, value) in items.iter_mut().zip(values) {
+                    item.from_json_value(value)?;
+                }
+            }
+            (Ty::ByteArray(bytes), JsonValue::String(s)) => {
+                *bytes = s;
+            }
+            _ => return Err(PrimitiveError::TypeMismatch),
+        }
+        Ok(())
+    }
 }
 
 #[derive(Debug)]

crates/torii/core/src/executor/mod.rs

@@ -548,14 +548,7 @@ impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
                 if em_query.is_historical {
                     event_counter += 1;
 
-                    let data = em_query
-                        .ty
-                        .serialize()?
-                        .iter()
-                        .map(|felt| format!("{:#x}", felt))
-                        .collect::<Vec<String>>()
-                        .join("/");
-
+                    let data = serde_json::to_string(&em_query.ty.to_json_value()?)?;
                     sqlx::query(
                         "INSERT INTO event_messages_historical (id, keys, event_id, data, \
                          model_id, executed_at) VALUES (?, ?, ?, ?, ?, ?) RETURNING *",

crates/torii/grpc/src/server/mod.rs

@@ -42,7 +42,6 @@ use tonic_web::GrpcWebLayer;
 use torii_core::error::{Error, ParseError, QueryError};
 use torii_core::model::{build_sql_query, map_row_to_ty};
 use torii_core::sql::cache::ModelCache;
-use torii_core::sql::utils::sql_string_to_felts;
 use torii_core::types::{Token, TokenBalance};
 use tower_http::cors::{AllowOrigin, CorsLayer};
 
@@ -368,7 +367,8 @@ impl DojoWorld {
                 .model(&Felt::from_str(&model_id).map_err(ParseError::FromStr)?)
                 .await?;
             let mut schema = model.schema;
-            schema.deserialize(&mut sql_string_to_felts(&data))?;
+            schema
+                .from_json_value(serde_json::from_str(&data).map_err(ParseError::FromJsonStr)?)?;
 
             let entity = entities
                 .entry(id)